From 762a2a99fce0b5799a067ef597310cd9296b24fc Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Thu, 4 Aug 2016 11:46:16 +0800 Subject: [PATCH 01/80] Fix `jl_static_show` for `bitstype` The byte gets sign extended when passing to the vararg `jl_printf` and then printed as an unsigned int which might come with unwanted `0xff` prefix... (cherry picked from commit bffa84e0c25492f15646586ace7e4bb77bb2916e) ref #17803 --- src/builtins.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/builtins.c b/src/builtins.c index b64c01be9611c..4698cab3afcbb 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -1479,7 +1479,7 @@ static size_t jl_static_show_x_(JL_STREAM *out, jl_value_t *v, jl_datatype_t *vt size_t nb = jl_datatype_size(vt); size_t tlen = jl_datatype_nfields(vt); if (nb > 0 && tlen == 0) { - char *data = (char*)jl_data_ptr(v); + uint8_t *data = (uint8_t*)v; n += jl_printf(out, "0x"); for(int i=nb-1; i >= 0; --i) n += jl_printf(out, "%02" PRIx8, data[i]); From c7cc66f1f6d25d47cc97c5b6d75dcb54fc71c754 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Wed, 10 Aug 2016 21:12:11 -0700 Subject: [PATCH 02/80] Revert "Revert "Merge pull request #17522 from JuliaLang/jn/reliable-flush-close"" This reverts commit 58a14f38a312cf61451849badeaf695b9e50299c. --- base/process.jl | 19 ++++++++---- base/socket.jl | 14 ++------- base/stream.jl | 31 ++++++++++++++++--- .../md5 | 1 + .../sha512 | 1 + .../md5 | 1 - .../sha512 | 1 - deps/libuv.version | 2 +- src/init.c | 9 +++++- src/jl_uv.c | 23 +++++++++++--- test/spawn.jl | 10 ++++-- 11 files changed, 78 insertions(+), 34 deletions(-) create mode 100644 deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/md5 create mode 100644 deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/sha512 delete mode 100644 deps/checksums/libuv-ecbd6eddfac4940ab8db57c73166a7378563ebd3.tar.gz/md5 delete mode 100644 deps/checksums/libuv-ecbd6eddfac4940ab8db57c73166a7378563ebd3.tar.gz/sha512 diff --git a/base/process.jl b/base/process.jl index 55495765febe5..88d51cb4f1634 100644 --- a/base/process.jl +++ b/base/process.jl @@ -289,7 +289,7 @@ type Process <: AbstractPipe typemin(fieldtype(Process, :termsignal)), false, Condition(), false, Condition()) finalizer(this, uvfinalize) - this + return this end end pipe_reader(p::Process) = p.out @@ -325,9 +325,12 @@ function _jl_spawn(cmd, argv, loop::Ptr{Void}, pp::Process, end function uvfinalize(proc::Process) - proc.handle != C_NULL && ccall(:jl_close_uv, Void, (Ptr{Void},), proc.handle) - disassociate_julia_struct(proc) - proc.handle = C_NULL + if proc.handle != C_NULL + disassociate_julia_struct(proc.handle) + ccall(:jl_close_uv, Void, (Ptr{Void},), proc.handle) + proc.handle = C_NULL + end + nothing end function uv_return_spawn(p::Ptr{Void}, exit_status::Int64, termsignal::Int32) @@ -336,7 +339,9 @@ function uv_return_spawn(p::Ptr{Void}, exit_status::Int64, termsignal::Int32) proc = unsafe_pointer_to_objref(data)::Process proc.exitcode = exit_status proc.termsignal = termsignal - if isa(proc.exitcb, Function) proc.exitcb(proc, exit_status, termsignal) end + if isa(proc.exitcb, Function) + proc.exitcb(proc, exit_status, termsignal) + end ccall(:jl_close_uv, Void, (Ptr{Void},), proc.handle) notify(proc.exitnotify) nothing @@ -344,7 +349,9 @@ end function _uv_hook_close(proc::Process) proc.handle = C_NULL - if isa(proc.closecb, Function) proc.closecb(proc) end + if isa(proc.closecb, Function) + proc.closecb(proc) + end notify(proc.closenotify) end diff --git a/base/socket.jl b/base/socket.jl index 29dd8d6caa3aa..8365f219fe711 100644 --- a/base/socket.jl +++ b/base/socket.jl @@ -281,7 +281,7 @@ function TCPSocket() throw(UVError("failed to create tcp socket",err)) end this.status = StatusInit - this + return this end type TCPServer <: LibuvServer @@ -312,7 +312,7 @@ function TCPServer() throw(UVError("failed to create tcp server",err)) end this.status = StatusInit - this + return this end isreadable(io::TCPSocket) = isopen(io) || nb_available(io) > 0 @@ -365,19 +365,11 @@ function UDPSocket() throw(UVError("failed to create udp socket",err)) end this.status = StatusInit - this + return this end show(io::IO, stream::UDPSocket) = print(io, typeof(stream), "(", uv_status_string(stream), ")") -function uvfinalize(uv::Union{TTY,PipeEndpoint,PipeServer,TCPServer,TCPSocket,UDPSocket}) - if (uv.status != StatusUninit && uv.status != StatusInit) - close(uv) - end - disassociate_julia_struct(uv) - uv.handle = C_NULL -end - function _uv_hook_close(sock::UDPSocket) sock.handle = C_NULL sock.status = StatusClosed diff --git a/base/stream.jl b/base/stream.jl index d649b88116736..733bba8d450ce 100644 --- a/base/stream.jl +++ b/base/stream.jl @@ -322,9 +322,26 @@ function wait_close(x::Union{LibuvStream, LibuvServer}) end function close(stream::Union{LibuvStream, LibuvServer}) - if isopen(stream) && stream.status != StatusClosing - ccall(:jl_close_uv,Void, (Ptr{Void},), stream.handle) - stream.status = StatusClosing + if isopen(stream) + if stream.status != StatusClosing + ccall(:jl_close_uv, Void, (Ptr{Void},), stream.handle) + stream.status = StatusClosing + end + if uv_handle_data(stream) != C_NULL + stream_wait(stream, stream.closenotify) + end + end + nothing +end + +function uvfinalize(uv::Union{LibuvStream, LibuvServer}) + if uv.handle != C_NULL + disassociate_julia_struct(uv.handle) # not going to call the usual close hooks + if uv.status != StatusUninit && uv.status != StatusInit + close(uv) + uv.handle = C_NULL + uv.status = StatusClosed + end end nothing end @@ -472,8 +489,10 @@ function uv_readcb(handle::Ptr{Void}, nread::Cssize_t, buf::Ptr{Void}) stream.status = StatusEOF # libuv called stop_reading already notify(stream.readnotify) notify(stream.closenotify) - else - close(stream) + elseif stream.status != StatusClosing + # begin shutdown of the stream + ccall(:jl_close_uv, Void, (Ptr{Void},), stream.handle) + stream.status = StatusClosing end else # This is a fatal connection error. Shutdown requests as per the usual @@ -1019,6 +1038,8 @@ function close(s::BufferStream) notify(s.close_c; all=true) nothing end +uvfinalize(s::BufferStream) = nothing + read(s::BufferStream, ::Type{UInt8}) = (wait_readnb(s, 1); read(s.buffer, UInt8)) unsafe_read(s::BufferStream, a::Ptr{UInt8}, nb::UInt) = (wait_readnb(s, Int(nb)); unsafe_read(s.buffer, a, nb)) nb_available(s::BufferStream) = nb_available(s.buffer) diff --git a/deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/md5 b/deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/md5 new file mode 100644 index 0000000000000..793f153e26212 --- /dev/null +++ b/deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/md5 @@ -0,0 +1 @@ +c6a019d79d20eabc39619a04961c9a3b diff --git a/deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/sha512 b/deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/sha512 new file mode 100644 index 0000000000000..f3f1ec9dca483 --- /dev/null +++ b/deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/sha512 @@ -0,0 +1 @@ +478ab473244b01bef344892a75e09fef50da8fb1a7212e0257c53f3223de4fde5f6bd449eef34bc1f025481c7d9f854002acb6eb203b447a50a34bae4ad9dee4 diff --git a/deps/checksums/libuv-ecbd6eddfac4940ab8db57c73166a7378563ebd3.tar.gz/md5 b/deps/checksums/libuv-ecbd6eddfac4940ab8db57c73166a7378563ebd3.tar.gz/md5 deleted file mode 100644 index 6dfbbd2a475fb..0000000000000 --- a/deps/checksums/libuv-ecbd6eddfac4940ab8db57c73166a7378563ebd3.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -7248e38acefd92761c54640622357b7b diff --git a/deps/checksums/libuv-ecbd6eddfac4940ab8db57c73166a7378563ebd3.tar.gz/sha512 b/deps/checksums/libuv-ecbd6eddfac4940ab8db57c73166a7378563ebd3.tar.gz/sha512 deleted file mode 100644 index e04d388782eba..0000000000000 --- a/deps/checksums/libuv-ecbd6eddfac4940ab8db57c73166a7378563ebd3.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -2fad17bddc568125d50ce50b7f27aa2de4380400589043d74b180883c96070c2dcad93557f2f5f5416cc297de3a66b35ec8942bd33d8ef1bb22744dad60b760d diff --git a/deps/libuv.version b/deps/libuv.version index efe634777261b..51f79989c5fc3 100644 --- a/deps/libuv.version +++ b/deps/libuv.version @@ -1,2 +1,2 @@ LIBUV_BRANCH=julia-uv1.9.0 -LIBUV_SHA1=ecbd6eddfac4940ab8db57c73166a7378563ebd3 +LIBUV_SHA1=cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68 diff --git a/src/init.c b/src/init.c index cb4758b8c56c0..a2d74fe60cc91 100644 --- a/src/init.c +++ b/src/init.c @@ -217,6 +217,7 @@ static struct uv_shutdown_queue_item *next_shutdown_queue_item(struct uv_shutdow void jl_init_timing(void); void jl_destroy_timing(void); +void jl_uv_call_close_callback(jl_value_t *val); JL_DLLEXPORT void jl_atexit_hook(int exitcode) { @@ -270,6 +271,13 @@ JL_DLLEXPORT void jl_atexit_hook(int exitcode) continue; } switch(handle->type) { + case UV_PROCESS: + // cause Julia to forget about the Process object + if (handle->data) + jl_uv_call_close_callback((jl_value_t*)handle->data); + // and make libuv think it is already dead + ((uv_process_t*)handle)->pid = 0; + // fall-through case UV_TTY: case UV_UDP: case UV_TCP: @@ -283,7 +291,6 @@ JL_DLLEXPORT void jl_atexit_hook(int exitcode) case UV_PREPARE: case UV_CHECK: case UV_SIGNAL: - case UV_PROCESS: case UV_FILE: // These will be shutdown as appropriate by jl_close_uv jl_close_uv(handle); diff --git a/src/jl_uv.c b/src/jl_uv.c index 3cdc6c61340c1..2c9f312ae682a 100644 --- a/src/jl_uv.c +++ b/src/jl_uv.c @@ -78,17 +78,17 @@ void jl_init_signal_async(void) } #endif -static void jl_uv_call_close_callback(jl_value_t *val) +void jl_uv_call_close_callback(jl_value_t *val) { jl_value_t *args[2]; args[0] = jl_get_global(jl_base_relative_to(((jl_datatype_t*)jl_typeof(val))->name->module), jl_symbol("_uv_hook_close")); // topmod(typeof(val))._uv_hook_close args[1] = val; assert(args[0]); - jl_apply(args, 2); + jl_apply(args, 2); // TODO: wrap in try-catch? } -JL_DLLEXPORT void jl_uv_closeHandle(uv_handle_t *handle) +static void jl_uv_closeHandle(uv_handle_t *handle) { // if the user killed a stdio handle, // revert back to direct stdio FILE* writes @@ -107,7 +107,7 @@ JL_DLLEXPORT void jl_uv_closeHandle(uv_handle_t *handle) free(handle); } -JL_DLLEXPORT void jl_uv_shutdownCallback(uv_shutdown_t *req, int status) +static void jl_uv_shutdownCallback(uv_shutdown_t *req, int status) { /* * This happens if the remote machine closes the connecition while we're @@ -180,8 +180,21 @@ JL_DLLEXPORT int jl_init_pipe(uv_pipe_t *pipe, int writable, int readable, return err; } +static void jl_proc_exit_cleanup(uv_process_t *process, int64_t exit_status, int term_signal) +{ + uv_close((uv_handle_t*)process, (uv_close_cb)&free); +} + JL_DLLEXPORT void jl_close_uv(uv_handle_t *handle) { + if (handle->type == UV_PROCESS && ((uv_process_t*)handle)->pid != 0) { + // take ownership of this handle, + // so we can waitpid for the resource to exit and avoid leaving zombies + assert(handle->data == NULL); // make sure Julia has forgotten about it already + ((uv_process_t*)handle)->exit_cb = jl_proc_exit_cleanup; + return; + } + if (handle->type == UV_FILE) { uv_fs_t req; jl_uv_file_t *fd = (jl_uv_file_t*)handle; @@ -230,7 +243,7 @@ JL_DLLEXPORT void jl_close_uv(uv_handle_t *handle) JL_DLLEXPORT void jl_forceclose_uv(uv_handle_t *handle) { - uv_close(handle,&jl_uv_closeHandle); + uv_close(handle, &jl_uv_closeHandle); } JL_DLLEXPORT void jl_uv_associate_julia_struct(uv_handle_t *handle, diff --git a/test/spawn.jl b/test/spawn.jl index 18433a6f83148..0040630cf2016 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -265,9 +265,9 @@ let bad = "bad\0name" end # issue #12829 -let out = Pipe(), echo = `$exename --startup-file=no -e 'print(STDOUT, " 1\t", readstring(STDIN))'`, ready = Condition() +let out = Pipe(), echo = `$exename --startup-file=no -e 'print(STDOUT, " 1\t", readstring(STDIN))'`, ready = Condition(), t @test_throws ArgumentError write(out, "not open error") - @async begin # spawn writer task + t = @async begin # spawn writer task open(echo, "w", out) do in1 open(echo, "w", out) do in2 notify(ready) @@ -282,10 +282,13 @@ let out = Pipe(), echo = `$exename --startup-file=no -e 'print(STDOUT, " 1\t", r @test isreadable(out) @test iswritable(out) close(out.in) + @test !isopen(out.in) + is_windows() || @test !isopen(out.out) # it takes longer to propagate EOF through the Windows event system @test_throws ArgumentError write(out, "now closed error") @test isreadable(out) @test !iswritable(out) - @test isopen(out) + is_windows() && Base.process_events(false) # should be enough steps to fully propagate EOF now + @test !isopen(out) end wait(ready) # wait for writer task to be ready before using `out` @test nb_available(out) == 0 @@ -308,6 +311,7 @@ let out = Pipe(), echo = `$exename --startup-file=no -e 'print(STDOUT, " 1\t", r @test isempty(read(out)) @test eof(out) @test desc == "Pipe(open => active, 0 bytes waiting)" + wait(t) end # issue #8529 From 7306033603e6cfdc0f09e001f5bffc005d3b73d9 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Thu, 4 Aug 2016 07:28:31 -0700 Subject: [PATCH 03/80] Fix #17805, spawn test failure on Windows introduced by #17522 (cherry picked from commit a3f288ce54ccf66ffaebb7d31574b1ed552e0045) ref #17815 --- test/spawn.jl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/spawn.jl b/test/spawn.jl index 0040630cf2016..7fa04587e3b28 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -287,7 +287,12 @@ let out = Pipe(), echo = `$exename --startup-file=no -e 'print(STDOUT, " 1\t", r @test_throws ArgumentError write(out, "now closed error") @test isreadable(out) @test !iswritable(out) - is_windows() && Base.process_events(false) # should be enough steps to fully propagate EOF now + if is_windows() + # WINNT kernel does not provide a fast mechanism for async propagation + # of EOF for a blocking stream, so just wait for it to catch up. + # This shouldn't take much more than 32ms. + Base.wait_close(out) + end @test !isopen(out) end wait(ready) # wait for writer task to be ready before using `out` From c573b7b5fd623202ab7a1b6215dd49cb984d8952 Mon Sep 17 00:00:00 2001 From: jw3126 Date: Thu, 4 Aug 2016 15:51:55 +0200 Subject: [PATCH 04/80] Fixed doc/conf.py #17813 Fixed missing UnicodeCharacters doc/conf.py, see #17813 (cherry picked from commit f566f0d254e13d04bc409c6d41e0272c18ba1fbc) ref #17814 remove patch copy-paste + artifacts from doc/conf.py (cherry picked from commit 6ba6d8dd4d053934258bbe2ff6f3466d5114cb72) --- doc/conf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/conf.py b/doc/conf.py index 4d99c8104f05e..934e10d79bb41 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -201,6 +201,7 @@ \DeclareUnicodeCharacter{025B}{\ensuremath{\varepsilon}} \DeclareUnicodeCharacter{03B3}{\ensuremath{\gamma}} \DeclareUnicodeCharacter{03B8}{\ensuremath{\theta}} + \DeclareUnicodeCharacter{03BD}{\ensuremath{\nu}} \DeclareUnicodeCharacter{03C0}{\ensuremath{\pi}} \DeclareUnicodeCharacter{03C6}{\ensuremath{\varphi}} \DeclareUnicodeCharacter{1D34}{\ensuremath{^{\mathrm{H}}}} @@ -213,6 +214,7 @@ \DeclareUnicodeCharacter{2209}{\ensuremath{\notin}} \DeclareUnicodeCharacter{220C}{\ensuremath{\not\ni}} \DeclareUnicodeCharacter{2211}{\ensuremath{\sum}} + \DeclareUnicodeCharacter{2213}{\ensuremath{\mp}} \DeclareUnicodeCharacter{221A}{\ensuremath{\sqrt{}}} \DeclareUnicodeCharacter{221B}{\ensuremath{\sqrt[3]{}}} \DeclareUnicodeCharacter{222A}{\ensuremath{\cup}} From 3d91b01c0d8ecde9bf2a1c3e6e6643925f2ad888 Mon Sep 17 00:00:00 2001 From: kshyatt Date: Wed, 3 Aug 2016 11:12:52 -0700 Subject: [PATCH 05/80] Moved docs out of helpDB, way more examples Lots of our documentation for some relatively simple iterator methods had no examples. I added a bunch, moved docstrings out of HelpDB, and tried to illustrate some common use cases. (cherry picked from commit ad5d244934592b721bcc665db7a447ae8fb87625) ref #17790 --- base/abstractarray.jl | 10 ++ base/array.jl | 95 ++++++++++++++++- base/docs/helpdb/Base.jl | 178 ------------------------------- base/iterator.jl | 124 +++++++++++++++++++++- base/operators.jl | 10 ++ base/range.jl | 18 ++++ base/reduce.jl | 97 +++++++++++++++++ doc/stdlib/collections.rst | 207 ++++++++++++++++++++++++++++++++++++- doc/stdlib/math.rst | 5 + 9 files changed, 562 insertions(+), 182 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 22714acac9d16..4f9c4880163ee 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1338,6 +1338,16 @@ Call function `f` on each element of iterable `c`. For multiple iterable arguments, `f` is called elementwise. `foreach` should be used instead of `map` when the results of `f` are not needed, for example in `foreach(println, array)`. + +```jldoctest +julia> a +1:3:7 + +julia> foreach(x->println(x^2),a) +1 +16 +49 +``` """ foreach(f) = (f(); nothing) foreach(f, itr) = (for x in itr; f(x); end; nothing) diff --git a/base/array.jl b/base/array.jl index e4d98d799b3c8..079bdfd1e5d99 100644 --- a/base/array.jl +++ b/base/array.jl @@ -824,6 +824,17 @@ function findnz{T}(A::AbstractMatrix{T}) return (I, J, NZs) end +""" + findmax(itr) -> (x, index) + +Returns the maximum element and its index. +The collection must not be empty. + +```jldoctest +julia> findmax([8,0.1,-9,pi]) +(8.0,1) +``` +""" function findmax(a) if isempty(a) throw(ArgumentError("collection must be non-empty")) @@ -842,6 +853,17 @@ function findmax(a) return (m, mi) end +""" + findmin(itr) -> (x, index) + +Returns the minimum element and its index. +The collection must not be empty. + +```jldoctest +julia> findmax([8,0.1,-9,pi]) +(-9.0,3) +``` +""" function findmin(a) if isempty(a) throw(ArgumentError("collection must be non-empty")) @@ -860,16 +882,87 @@ function findmin(a) return (m, mi) end +""" + indmax(itr) -> Integer + +Returns the index of the maximum element in a collection. +```jldoctest +julia> indmax([8,0.1,-9,pi]) +1 +``` +""" indmax(a) = findmax(a)[2] + +""" + indmin(itr) -> Integer + +Returns the index of the minimum element in a collection. +```jldoctest +julia> indmin([8,0.1,-9,pi]) +3 +``` +""" indmin(a) = findmin(a)[2] # similar to Matlab's ismember -# returns a vector containing the highest index in b for each value in a that is a member of b +""" + indexin(a, b) + +Returns a vector containing the highest index in `b` for +each value in `a` that is a member of `b` . The output +vector contains 0 wherever `a` is not a member of `b`. + +```jldoctest +julia> a = ['a', 'b', 'c', 'b', 'd', 'a']; + +julia> b = ['a','b','c'] + +julia> indexin(a,b) +6-element Array{Int64,1}: + 1 + 2 + 3 + 2 + 0 + 1 + +julia> indexin(b,a) +3-element Array{Int64,1}: + 6 + 4 + 3 +``` +""" function indexin(a::AbstractArray, b::AbstractArray) bdict = Dict(zip(b, 1:length(b))) [get(bdict, i, 0) for i in a] end +""" + findin(a, b) + +Returns the indices of elements in collection `a` that appear in collection `b`. + +```jldoctest +julia> a = collect(1:3:15) +5-element Array{Int64,1}: + 1 + 4 + 7 + 10 + 13 + +julia> b = collect(2:4:10) +3-element Array{Int64,1}: + 2 + 6 + 10 + +julia> findin(a,b) # 10 is the only common element +1-element Array{Int64,1}: + 4 +``` +""" function findin(a, b) ind = Array{Int,1}(0) bset = Set(b) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 9b11cbed6995c..9599c3a57dbe6 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -79,13 +79,6 @@ the woken task. """ schedule -""" - step(r) - -Get the step size of a [`Range`](:obj:`Range`) object. -""" -step - """ takebuf_array(b::IOBuffer) @@ -113,13 +106,6 @@ set of characters) is provided, instead remove characters contained in it. """ lstrip -""" - indmin(itr) -> Integer - -Returns the index of the minimum element in a collection. -""" -indmin - """ powermod(x, p, m) @@ -1213,14 +1199,6 @@ greater than 1, and `x` must not be less than 1. """ prevpow -""" - indexin(a, b) - -Returns a vector containing the highest index in `b` for each value in `a` that is a member -of `b` . The output vector contains 0 wherever `a` is not a member of `b`. -""" -indexin - """ permutedims(A, perm) @@ -1376,13 +1354,6 @@ Like `randsubseq`, but the results are stored in `S` (which is resized as needed """ randsubseq! -""" - maximum(itr) - -Returns the largest element in a collection. -""" -maximum(itr) - """ maximum(A, dims) @@ -2473,13 +2444,6 @@ Largest integer less than or equal to `x/y`. """ fld -""" - indmax(itr) -> Integer - -Returns the index of the maximum element in a collection. -""" -indmax - """ writecsv(filename, A) @@ -3009,13 +2973,6 @@ Get the concrete type of `x`. """ typeof -""" - drop(iter, n) - -An iterator that generates all but the first `n` elements of `iter`. -""" -drop - """ acsc(x) @@ -3170,13 +3127,6 @@ Compute the inverse error complementary function of a real `x`, defined by """ erfcinv -""" - minabs(itr) - -Compute the minimum absolute value of a collection of values. -""" -minabs(itr) - """ minabs(A, dims) @@ -3559,16 +3509,6 @@ handle properly. """ OutOfMemoryError -""" - zip(iters...) - -For a set of iterable objects, returns an iterable of tuples, where the `i`th tuple contains -the `i`th component of each input iterable. - -Note that [`zip`](:func:`zip`) is its own inverse: `collect(zip(zip(a...)...)) == collect(a)`. -""" -zip - """ SystemError(prefix::AbstractString, [errno::Int32]) @@ -4647,13 +4587,6 @@ Compute the inverse hyperbolic sine of `x`. """ asinh -""" - count(p, itr) -> Integer - -Count the number of elements in `itr` for which predicate `p` returns `true`. -""" -count - """ atreplinit(f) @@ -4672,20 +4605,6 @@ or vector or set of characters) is provided, instead remove characters contained """ strip -""" - findin(a, b) - -Returns the indices of elements in collection `a` that appear in collection `b`. -""" -findin - -""" - minimum(itr) - -Returns the smallest element in a collection. -""" -minimum(itr) - """ minimum(A, dims) @@ -4741,13 +4660,6 @@ Like redirect_stdout, but for STDIN. Note that the order of the return tuple is """ redirect_stdin -""" - minmax(x, y) - -Return `(min(x,y), max(x,y))`. See also: [`extrema`](:func:`extrema`) that returns `(minimum(x), maximum(x))`. -""" -minmax - """ mktemp([parent=tempdir()]) @@ -4884,13 +4796,6 @@ last argument optionally specifies a size beyond which the buffer may not be gro """ IOBuffer(data=?) -""" - findmax(itr) -> (x, index) - -Returns the maximum element and its index. -""" -findmax(itr) - """ findmax(A, dims) -> (maxval, index) @@ -5502,13 +5407,6 @@ The highest value representable by the given (real) numeric `DataType`. """ typemax -""" - all(itr) -> Bool - -Test whether all elements of a boolean collection are `true`. -""" -all(itr) - """ all(A, dims) @@ -5516,18 +5414,6 @@ Test whether all values along the given dimensions of an array are `true`. """ all(A::AbstractArray, dims) -""" - all(p, itr) -> Bool - -Determine whether predicate `p` returns `true` for all elements of `itr`. - -```jldoctest -julia> all(i->(4<=i<=6), [4,5,6]) -true -``` -""" -all(p, itr) - """ bind(socket::Union{UDPSocket, TCPSocket}, host::IPAddr, port::Integer; ipv6only=false) @@ -6968,13 +6854,6 @@ Compute the trigamma function of `x` (the logarithmic second derivative of `gamm """ trigamma -""" - findmin(itr) -> (x, index) - -Returns the minimum element and its index. -""" -findmin(itr) - """ findmin(A, dims) -> (minval, index) @@ -7472,13 +7351,6 @@ throwing an `ArgumentError` indicating the position of the first non-ASCII byte. """ ascii(s) -""" - maxabs(itr) - -Compute the maximum absolute value of a collection of values. -""" -maxabs(itr) - """ maxabs(A, dims) @@ -8074,13 +7946,6 @@ Get the additive identity element for the type of `x` (`x` can also specify the """ zero -""" - any(itr) -> Bool - -Test whether any elements of a boolean collection are `true`. -""" -any(itr) - """ any(A, dims) @@ -8088,13 +7953,6 @@ Test whether any values along the given dimensions of an array are `true`. """ any(::AbstractArray,dims) -""" - any(p, itr) -> Bool - -Determine whether predicate `p` returns `true` for any elements of `itr`. -""" -any(p,itr) - """ cosc(x) @@ -8230,13 +8088,6 @@ break identities such as `(x-y==0) == (x==y)`. """ set_zero_subnormals -""" - take(iter, n) - -An iterator that generates at most the first `n` elements of `iter`. -""" -take - """ frexp(val) @@ -8827,14 +8678,6 @@ julia> A """ pop!(collection) -""" - filter(function, collection) - -Return a copy of `collection`, removing elements for which `function` is `false`. For -associative collections, the function is passed two arguments (key and value). -""" -filter - """ randperm([rng,] n) @@ -8930,27 +8773,6 @@ Remove a single trailing newline from a string. """ chomp -""" - enumerate(iter) - -An iterator that yields `(i, x)` where `i` is an index starting at 1, and -`x` is the `i`th value from the given iterator. It's useful when you need -not only the values `x` over which you are iterating, but also the index `i` -of the iterations. - -```jldoctest -julia> a = ["a", "b", "c"]; - -julia> for (index, value) in enumerate(a) - println("\$index \$value") - end -1 a -2 b -3 c -``` -""" -enumerate - """ >=(x, y) ≥(x,y) diff --git a/base/iterator.jl b/base/iterator.jl index 56c97b7348618..6c93bcbb6be37 100644 --- a/base/iterator.jl +++ b/base/iterator.jl @@ -17,7 +17,27 @@ _diff_length(a, b, A, B) = max(length(a)-length(b), 0) immutable Enumerate{I} itr::I end -enumerate(itr) = Enumerate(itr) + +""" + enumerate(iter) + +An iterator that yields `(i, x)` where `i` is an index starting at 1, and +`x` is the `i`th value from the given iterator. It's useful when you need +not only the values `x` over which you are iterating, but also the index `i` +of the iterations. + +```jldoctest +julia> a = ["a", "b", "c"]; + +julia> for (index, value) in enumerate(a) + println("\$index \$value") + end +1 a +2 b +3 c +``` +""" +enumerate(iter) = Enumerate(iter) length(e::Enumerate) = length(e.itr) size(e::Enumerate) = size(e.itr) @@ -83,6 +103,37 @@ immutable Zip{I, Z<:AbstractZipIterator} <: AbstractZipIterator a::I z::Z end + +""" + zip(iters...) + +For a set of iterable objects, returns an iterable of tuples, where the `i`th tuple contains +the `i`th component of each input iterable. + +Note that [`zip`](:func:`zip`) is its own inverse: `collect(zip(zip(a...)...)) == collect(a)`. + +```jldoctest +julia> a = 1:5 +1:5 + +julia> b = ["e","d","b","c","a"] +5-element Array{String,1}: + "e" + "d" + "b" + "c" + "a" + +julia> c = zip(a,b) +Base.Zip2{UnitRange{Int64},Array{String,1}}(1:5,String["e","d","b","c","a"]) + +julia> length(c) +5 + +julia> first(c) +(1,"e") +``` +""" zip(a, b, c...) = Zip(a, zip(b, c...)) length(z::Zip) = _min_length(z.a, z.z, iteratorsize(z.a), iteratorsize(z.z)) size(z::Zip) = promote_shape(size(z.a), size(z.z)) @@ -109,6 +160,26 @@ immutable Filter{F,I} flt::F itr::I end + +""" + filter(function, collection) + +Return a copy of `collection`, removing elements for which `function` is `false`. For +associative collections, the function is passed two arguments (key and value). + +```jldocttest +julia> a = 1:10 +1:10 + +julia> filter(isodd, a) +5-element Array{Int64,1}: + 1 + 3 + 5 + 7 + 9 +``` +""" filter(flt, itr) = Filter(flt, itr) start(f::Filter) = start_filter(f.flt, f.itr) @@ -204,6 +275,32 @@ immutable Take{I} xs::I n::Int end + +""" + take(iter, n) + +An iterator that generates at most the first `n` elements of `iter`. + +```jldoctest +julia> a = 1:2:11 +1:2:11 + +julia> collect(a) +6-element Array{Int64,1}: + 1 + 3 + 5 + 7 + 9 + 11 + +julia> collect(take(a,3)) +3-element Array{Int64,1}: + 1 + 3 + 5 +``` +""" take(xs, n::Int) = Take(xs, n) eltype{I}(::Type{Take{I}}) = eltype(I) @@ -232,6 +329,31 @@ immutable Drop{I} xs::I n::Int end + +""" + drop(iter, n) + +An iterator that generates all but the first `n` elements of `iter`. + +```jldoctest +julia> a = 1:2:11 +1:2:11 + +julia> collect(a) +6-element Array{Int64,1}: + 1 + 3 + 5 + 7 + 9 + 11 + +julia> collect(drop(a,4)) +2-element Array{Int64,1}: + 9 + 11 +``` +""" drop(xs, n::Int) = Drop(xs, n) eltype{I}(::Type{Drop{I}}) = eltype(I) diff --git a/base/operators.jl b/base/operators.jl index 1998122216ba1..31faf73b1d77f 100644 --- a/base/operators.jl +++ b/base/operators.jl @@ -81,6 +81,16 @@ cmp(x::Integer, y::Integer) = ifelse(isless(x,y), -1, ifelse(isless(y,x), 1, 0)) max(x,y) = ifelse(y < x, x, y) min(x,y) = ifelse(y < x, y, x) +""" + minmax(x, y) + +Return `(min(x,y), max(x,y))`. See also: [`extrema`](:func:`extrema`) that returns `(minimum(x), maximum(x))`. + +```jldoctest +julia> minmax('c','b') +('b','c') +``` +""" minmax(x,y) = y < x ? (y, x) : (x, y) scalarmax(x,y) = max(x,y) diff --git a/base/range.jl b/base/range.jl index 5c60b3078edc1..caca080802553 100644 --- a/base/range.jl +++ b/base/range.jl @@ -332,6 +332,24 @@ isempty(r::AbstractUnitRange) = first(r) > last(r) isempty(r::FloatRange) = length(r) == 0 isempty(r::LinSpace) = length(r) == 0 +""" + step(r) + +Get the step size of a [`Range`](:obj:`Range`) object. +```jldoctest +julia> step(1:10) +1 + +julia> step(1:2:10) +2 + +julia> step(2.5:0.3:10.9) +0.3 + +julia> step(linspace(2.5,10.9,85)) +0.1 +``` +""" step(r::StepRange) = r.step step(r::AbstractUnitRange) = 1 step(r::FloatRange) = r.step/r.divisor diff --git a/base/reduce.jl b/base/reduce.jl index 4d26a67eee985..18ef01b4d63f4 100644 --- a/base/reduce.jl +++ b/base/reduce.jl @@ -286,10 +286,58 @@ end maximum(f::Callable, a) = mapreduce(f, scalarmax, a) minimum(f::Callable, a) = mapreduce(f, scalarmin, a) +""" + maximum(itr) + +Returns the largest element in a collection. + +```jldoctest +julia> maximum(-20.5:10) +9.5 + +julia> maximum([1,2,3]) +3 +``` +""" maximum(a) = mapreduce(identity, scalarmax, a) + +""" + minimum(itr) + +Returns the smallest element in a collection. + +```jldoctest +julia> minimum(-20.5:10) +-20.5 + +julia> minimum([1,2,3]) +1 +``` +""" minimum(a) = mapreduce(identity, scalarmin, a) +""" + maxabs(itr) + +Compute the maximum absolute value of a collection of values. + +```jldoctest +julia> maxabs([-1, 3, 4*im]) +4.0 +``` +""" maxabs(a) = mapreduce(abs, scalarmax, a) + +""" + minabs(itr) + +Compute the minimum absolute value of a collection of values. + +```jldoctest +julia> minabs([-1, 3, 4*im]) +1.0 +``` +""" minabs(a) = mapreduce(abs, scalarmin, a) ## extrema @@ -301,6 +349,14 @@ extrema(x::Real) = (x, x) extrema(itr) -> Tuple Compute both the minimum and maximum element in a single pass, and return them as a 2-tuple. + +```jldoctest +julia> extrema(2:10) +(2,10) + +julia> extrema([9,pi,4.5]) +(3.141592653589793,9.0) +``` """ function extrema(itr) s = start(itr) @@ -362,7 +418,18 @@ end ## all & any +""" + any(itr) -> Bool + +Test whether any elements of a boolean collection are `true`. +""" any(itr) = any(identity, itr) + +""" + all(itr) -> Bool + +Test whether all elements of a boolean collection are `true`. +""" all(itr) = all(identity, itr) nonboolean_error(f, op) = throw(ArgumentError(""" @@ -375,6 +442,16 @@ or_bool_only(a::Bool, b::Bool) = a|b and_bool_only(a, b) = nonboolean_error(:all, :&) and_bool_only(a::Bool, b::Bool) = a&b +""" + any(p, itr) -> Bool + +Determine whether predicate `p` returns `true` for any elements of `itr`. + +```jldoctest +julia> any(i->(4<=i<=6), [3,5,7]) +true +``` +""" any(f::Any, itr) = any(Predicate(f), itr) any(f::Predicate, itr) = mapreduce_sc_impl(f, |, itr) any(f::typeof(identity), itr) = @@ -382,6 +459,16 @@ any(f::typeof(identity), itr) = mapreduce_sc_impl(f, |, itr) : reduce(or_bool_only, itr) +""" + all(p, itr) -> Bool + +Determine whether predicate `p` returns `true` for all elements of `itr`. + +```jldoctest +julia> all(i->(4<=i<=6), [4,5,6]) +true +``` +""" all(f::Any, itr) = all(Predicate(f), itr) all(f::Predicate, itr) = mapreduce_sc_impl(f, &, itr) all(f::typeof(identity), itr) = @@ -408,6 +495,16 @@ end ## countnz & count +""" + count(p, itr) -> Integer + +Count the number of elements in `itr` for which predicate `p` returns `true`. + +```jldoctest +julia> count(i->(4<=i<=6), [2,3,4,5,6]) + 3 +``` +""" function count(pred, itr) n = 0 for x in itr diff --git a/doc/stdlib/collections.rst b/doc/stdlib/collections.rst index ad81ed3f37987..4cc7c6eb4be22 100644 --- a/doc/stdlib/collections.rst +++ b/doc/stdlib/collections.rst @@ -55,6 +55,28 @@ type. Note that :func:`zip` is its own inverse: ``collect(zip(zip(a...)...)) == collect(a)``\ . + .. doctest:: + + julia> a = 1:5 + 1:5 + + julia> b = ["e","d","b","c","a"] + 5-element Array{String,1}: + "e" + "d" + "b" + "c" + "a" + + julia> c = zip(a,b) + Base.Zip2{UnitRange{Int64},Array{String,1}}(1:5,String["e","d","b","c","a"]) + + julia> length(c) + 5 + + julia> first(c) + (1,"e") + .. function:: enumerate(iter) .. Docstring generated from Julia source @@ -90,12 +112,51 @@ type. An iterator that generates at most the first ``n`` elements of ``iter``\ . + .. doctest:: + + julia> a = 1:2:11 + 1:2:11 + + julia> collect(a) + 6-element Array{Int64,1}: + 1 + 3 + 5 + 7 + 9 + 11 + + julia> collect(take(a,3)) + 3-element Array{Int64,1}: + 1 + 3 + 5 + .. function:: drop(iter, n) .. Docstring generated from Julia source An iterator that generates all but the first ``n`` elements of ``iter``\ . + .. doctest:: + + julia> a = 1:2:11 + 1:2:11 + + julia> collect(a) + 6-element Array{Int64,1}: + 1 + 3 + 5 + 7 + 9 + 11 + + julia> collect(drop(a,4)) + 2-element Array{Int64,1}: + 9 + 11 + .. function:: cycle(iter) .. Docstring generated from Julia source @@ -230,12 +291,53 @@ Iterable Collections Returns a vector containing the highest index in ``b`` for each value in ``a`` that is a member of ``b`` . The output vector contains 0 wherever ``a`` is not a member of ``b``\ . + .. doctest:: + + julia> a = ['a', 'b', 'c', 'b', 'd', 'a']; + + julia> b = ['a','b','c'] + + julia> indexin(a,b) + 6-element Array{Int64,1}: + 1 + 2 + 3 + 2 + 0 + 1 + + julia> indexin(b,a) + 3-element Array{Int64,1}: + 6 + 4 + 3 + .. function:: findin(a, b) .. Docstring generated from Julia source Returns the indices of elements in collection ``a`` that appear in collection ``b``\ . + .. doctest:: + + julia> a = collect(1:3:15) + 5-element Array{Int64,1}: + 1 + 4 + 7 + 10 + 13 + + julia> b = collect(2:4:10) + 3-element Array{Int64,1}: + 2 + 6 + 10 + + julia> findin(a,b) # 10 is the only common element + 1-element Array{Int64,1}: + 4 + .. function:: unique(itr[, dim]) .. Docstring generated from Julia source @@ -308,6 +410,14 @@ Iterable Collections Returns the largest element in a collection. + .. doctest:: + + julia> maximum(-20.5:10) + 9.5 + + julia> maximum([1,2,3]) + 3 + .. function:: maximum(A, dims) .. Docstring generated from Julia source @@ -326,6 +436,14 @@ Iterable Collections Returns the smallest element in a collection. + .. doctest:: + + julia> minimum(-20.5:10) + -20.5 + + julia> minimum([1,2,3]) + 1 + .. function:: minimum(A, dims) .. Docstring generated from Julia source @@ -344,6 +462,14 @@ Iterable Collections Compute both the minimum and maximum element in a single pass, and return them as a 2-tuple. + .. doctest:: + + julia> extrema(2:10) + (2,10) + + julia> extrema([9,pi,4.5]) + (3.141592653589793,9.0) + .. function:: extrema(A,dims) -> Array{Tuple} .. Docstring generated from Julia source @@ -356,17 +482,32 @@ Iterable Collections Returns the index of the maximum element in a collection. + .. doctest:: + + julia> indmax([8,0.1,-9,pi]) + 1 + .. function:: indmin(itr) -> Integer .. Docstring generated from Julia source Returns the index of the minimum element in a collection. + .. doctest:: + + julia> indmin([8,0.1,-9,pi]) + 3 + .. function:: findmax(itr) -> (x, index) .. Docstring generated from Julia source - Returns the maximum element and its index. + Returns the maximum element and its index. The collection must not be empty. + + .. doctest:: + + julia> findmax([8,0.1,-9,pi]) + (8.0,1) .. function:: findmax(A, dims) -> (maxval, index) @@ -378,7 +519,12 @@ Iterable Collections .. Docstring generated from Julia source - Returns the minimum element and its index. + Returns the minimum element and its index. The collection must not be empty. + + .. doctest:: + + julia> findmax([8,0.1,-9,pi]) + (-9.0,3) .. function:: findmin(A, dims) -> (minval, index) @@ -404,6 +550,11 @@ Iterable Collections Compute the maximum absolute value of a collection of values. + .. doctest:: + + julia> maxabs([-1, 3, 4*im]) + 4.0 + .. function:: maxabs(A, dims) .. Docstring generated from Julia source @@ -422,6 +573,11 @@ Iterable Collections Compute the minimum absolute value of a collection of values. + .. doctest:: + + julia> minabs([-1, 3, 4*im]) + 1.0 + .. function:: minabs(A, dims) .. Docstring generated from Julia source @@ -554,12 +710,22 @@ Iterable Collections Count the number of elements in ``itr`` for which predicate ``p`` returns ``true``\ . + .. doctest:: + + julia> count(i->(4<=i<=6), [2,3,4,5,6]) + 3 + .. function:: any(p, itr) -> Bool .. Docstring generated from Julia source Determine whether predicate ``p`` returns ``true`` for any elements of ``itr``\ . + .. doctest:: + + julia> any(i->(4<=i<=6), [3,5,7]) + true + .. function:: all(p, itr) -> Bool .. Docstring generated from Julia source @@ -577,6 +743,16 @@ Iterable Collections Call function ``f`` on each element of iterable ``c``\ . For multiple iterable arguments, ``f`` is called elementwise. ``foreach`` should be used instead of ``map`` when the results of ``f`` are not needed, for example in ``foreach(println, array)``\ . + .. doctest:: + + julia> a + 1:3:7 + + julia> foreach(x->println(x^2),a) + 1 + 16 + 49 + .. function:: map(f, c...) -> collection .. Docstring generated from Julia source @@ -672,6 +848,20 @@ Iterable Collections Get the step size of a :obj:`Range` object. + .. doctest:: + + julia> step(1:10) + 1 + + julia> step(1:2:10) + 2 + + julia> step(2.5:0.3:10.9) + 0.3 + + julia> step(linspace(2.5,10.9,85)) + 0.1 + .. function:: collect(collection) .. Docstring generated from Julia source @@ -699,6 +889,19 @@ Iterable Collections Return a copy of ``collection``\ , removing elements for which ``function`` is ``false``\ . For associative collections, the function is passed two arguments (key and value). + .. code-block:: julia + + julia> a = 1:10 + 1:10 + + julia> filter(isodd, a) + 5-element Array{Int64,1}: + 1 + 3 + 5 + 7 + 9 + .. function:: filter!(function, collection) .. Docstring generated from Julia source diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index 6ff3d36e6afce..47620a1b53e14 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -1012,6 +1012,11 @@ Mathematical Functions Return ``(min(x,y), max(x,y))``\ . See also: :func:`extrema` that returns ``(minimum(x), maximum(x))``\ . + .. doctest:: + + julia> minmax('c','b') + ('b','c') + .. function:: clamp(x, lo, hi) .. Docstring generated from Julia source From 48f726730dc9eef4f1f2695d67a0411b4fe5dd95 Mon Sep 17 00:00:00 2001 From: Katharine Hyatt Date: Thu, 4 Aug 2016 16:55:55 -0700 Subject: [PATCH 06/80] Move array documentation out of HelpDB, add examples (#17795) Lots of examples for things that are probably confusing for new Julia users. Made `reducedim` and `mapreducedim` arguments more similar to the ones for `reduce` and `mapreduce`. (cherry picked from commit 3e00217d69fa3f59dfd1ee0a6fd04deccec0dd5b) --- base/abstractarray.jl | 43 ++++++++ base/arraymath.jl | 141 ++++++++++++++++++++++++ base/docs/helpdb/Base.jl | 95 ---------------- base/multidimensional.jl | 50 ++++++++- base/reducedim.jl | 54 ++++++++++ doc/stdlib/arrays.rst | 226 +++++++++++++++++++++++++++++++++++++-- 6 files changed, 506 insertions(+), 103 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 4f9c4880163ee..08d5956596f17 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1358,6 +1358,49 @@ foreach(f, itrs...) = (for z in zip(itrs...); f(z...); end; nothing) ## transform any set of dimensions ## dims specifies which dimensions will be transformed. for example ## dims==1:2 will call f on all slices A[:,:,...] +""" + mapslices(f, A, dims) + +Transform the given dimensions of array `A` using function `f`. `f` is called on each slice +of `A` of the form `A[...,:,...,:,...]`. `dims` is an integer vector specifying where the +colons go in this expression. The results are concatenated along the remaining dimensions. +For example, if `dims` is `[1,2]` and `A` is 4-dimensional, `f` is called on `A[:,:,i,j]` +for all `i` and `j`. + +```jldoctest +julia> a = reshape(collect(1:16),(2,2,2,2)) +2×2×2×2 Array{Int64,4}: +[:, :, 1, 1] = + 1 3 + 2 4 + +[:, :, 2, 1] = + 5 7 + 6 8 + +[:, :, 1, 2] = + 9 11 + 10 12 + +[:, :, 2, 2] = + 13 15 + 14 16 + +julia> mapslices(sum, a, [1,2]) +1×1×2×2 Array{Int64,4}: +[:, :, 1, 1] = + 10 + +[:, :, 2, 1] = + 26 + +[:, :, 1, 2] = + 42 + +[:, :, 2, 2] = + 58 +``` +""" mapslices(f, A::AbstractArray, dims) = mapslices(f, A, [dims...]) function mapslices(f, A::AbstractArray, dims::AbstractVector) if isempty(dims) diff --git a/base/arraymath.jl b/base/arraymath.jl index 319f7937cc8e4..6d3a2d9770413 100644 --- a/base/arraymath.jl +++ b/base/arraymath.jl @@ -157,6 +157,23 @@ function flipdim{T}(A::Array{T}, d::Integer) return B end +""" + rotl90(A) + +Rotate matrix `A` left 90 degrees. + +```jldoctest +julia> a = [1 2; 3 4] +2×2 Array{Int64,2}: + 1 2 + 3 4 + +julia> rotl90(a) +2×2 Array{Int64,2}: + 2 4 + 1 3 +``` +""" function rotl90(A::AbstractMatrix) ind1, ind2 = indices(A) B = similar(A, (ind2,ind1)) @@ -166,6 +183,24 @@ function rotl90(A::AbstractMatrix) end return B end + +""" + rotr90(A) + +Rotate matrix `A` right 90 degrees. + +```jldoctest +julia> a = [1 2; 3 4] +2×2 Array{Int64,2}: + 1 2 + 3 4 + +julia> rotr90(a) +2×2 Array{Int64,2}: + 3 1 + 4 2 +``` +""" function rotr90(A::AbstractMatrix) ind1, ind2 = indices(A) B = similar(A, (ind2,ind1)) @@ -175,6 +210,23 @@ function rotr90(A::AbstractMatrix) end return B end +""" + rot180(A) + +Rotate matrix `A` 180 degrees. + +```jldoctest +julia> a = [1 2; 3 4] +2×2 Array{Int64,2}: + 1 2 + 3 4 + +julia> rot180(a) +2×2 Array{Int64,2}: + 4 3 + 2 1 +``` +""" function rot180(A::AbstractMatrix) B = similar(A) ind1, ind2 = indices(A,1), indices(A,2) @@ -184,13 +236,102 @@ function rot180(A::AbstractMatrix) end return B end +""" + rotl90(A, k) + +Rotate matrix `A` left 90 degrees an integer `k` number of times. +If `k` is zero or a multiple of four, this is equivalent to a `copy`. + +```jldoctest +julia> a = [1 2; 3 4] +2×2 Array{Int64,2}: + 1 2 + 3 4 + +julia> rotl90(a,1) +2×2 Array{Int64,2}: + 2 4 + 1 3 + +julia> rotl90(a,2) +2×2 Array{Int64,2}: + 4 3 + 2 1 + +julia> rotl90(a,3) +2×2 Array{Int64,2}: + 3 1 + 4 2 + +julia> rotl90(a,4) +2×2 Array{Int64,2}: + 1 2 + 3 4 +``` +""" function rotl90(A::AbstractMatrix, k::Integer) k = mod(k, 4) k == 1 ? rotl90(A) : k == 2 ? rot180(A) : k == 3 ? rotr90(A) : copy(A) end +""" + rotr90(A, k) + +Rotate matrix `A` right 90 degrees an integer `k` number of times. If `k` is zero or a +multiple of four, this is equivalent to a `copy`. + +```jldoctest +julia> a = [1 2; 3 4] +2×2 Array{Int64,2}: + 1 2 + 3 4 + +julia> rotr90(a,1) +2×2 Array{Int64,2}: + 3 1 + 4 2 + +julia> rotr90(a,2) +2×2 Array{Int64,2}: + 4 3 + 2 1 + +julia> rotr90(a,3) +2×2 Array{Int64,2}: + 2 4 + 1 3 + +julia> rotr90(a,4) +2×2 Array{Int64,2}: + 1 2 + 3 4 +``` +""" rotr90(A::AbstractMatrix, k::Integer) = rotl90(A,-k) +""" + rot180(A, k) + +Rotate matrix `A` 180 degrees an integer `k` number of times. +If `k` is even, this is equivalent to a `copy`. + +```jldoctest +julia> a = [1 2; 3 4] +2×2 Array{Int64,2}: + 1 2 + 3 4 + +julia> rot180(a,1) +2×2 Array{Int64,2}: + 4 3 + 2 1 + +julia> rot180(a,2) +2×2 Array{Int64,2}: + 1 2 + 3 4 +``` +""" rot180(A::AbstractMatrix, k::Integer) = mod(k, 2) == 1 ? rot180(A) : copy(A) ## Transpose ## diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 9599c3a57dbe6..b8ab785fd9778 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -490,19 +490,6 @@ Get a backtrace object for the current program point. """ backtrace -""" - reducedim(f, A, dims[, initial]) - -Reduce 2-argument function `f` along dimensions of `A`. `dims` is a vector specifying the -dimensions to reduce, and `initial` is the initial value to use in the reductions. For `+`, `*`, -`max` and `min` the `initial` argument is optional. - -The associativity of the reduction is implementation-dependent; if you need a particular -associativity, e.g. left-to-right, you should write your own loop. See documentation for -`reduce`. -""" -reducedim - """ -(x) @@ -1105,17 +1092,6 @@ representable. """ ceil -""" - mapslices(f, A, dims) - -Transform the given dimensions of array `A` using function `f`. `f` is called on each slice -of `A` of the form `A[...,:,...,:,...]`. `dims` is an integer vector specifying where the -colons go in this expression. The results are concatenated along the remaining dimensions. -For example, if `dims` is `[1,2]` and `A` is 4-dimensional, `f` is called on `A[:,:,i,j]` -for all `i` and `j`. -""" -mapslices - """ issocket(path) -> Bool @@ -2321,15 +2297,6 @@ Bessel function of the second kind of order 1, ``Y_1(x)``. """ bessely1 -""" - cumprod(A, [dim]) - -Cumulative product along a dimension `dim` (defaults to 1). See also -[`cumprod!`](:func:`cumprod!`) to use a preallocated output array, both for performance and -to control the precision of the output (e.g. to avoid overflow). -""" -cumprod - """ besseljx(nu, x) @@ -3054,21 +3021,6 @@ block to create a new scope with copies of all variables referenced in the expre """ :@async -""" - rotr90(A) - -Rotate matrix `A` right 90 degrees. -""" -rotr90(A) - -""" - rotr90(A, k) - -Rotate matrix `A` right 90 degrees an integer `k` number of times. If `k` is zero or a -multiple of four, this is equivalent to a `copy`. -""" -rotr90(A, k) - """ readdir([dir]) -> Vector{String} @@ -3523,21 +3475,6 @@ Number of ways to choose `k` out of `n` items. """ binomial -""" - rot180(A) - -Rotate matrix `A` 180 degrees. -""" -rot180(A) - -""" - rot180(A, k) - -Rotate matrix `A` 180 degrees an integer `k` number of times. If `k` is even, this is -equivalent to a `copy`. -""" -rot180(A, k) - """ .<=(x, y) .≤(x,y) @@ -3903,21 +3840,6 @@ x == div(x,y)*y + rem(x,y) """ rem -""" - rotl90(A) - -Rotate matrix `A` left 90 degrees. -""" -rotl90(A) - -""" - rotl90(A, k) - -Rotate matrix `A` left 90 degrees an integer `k` number of times. If `k` is zero or a -multiple of four, this is equivalent to a `copy`. -""" -rotl90(A, k) - """ info(msg) @@ -5623,14 +5545,6 @@ handle comparison to other types via promotion rules where possible. """ Base.:(==) -""" - mapreducedim(f, op, A, dims[, initial]) - -Evaluates to the same as `reducedim(op, map(f, A), dims, f(initial))`, but is generally -faster because the intermediate array is avoided. -""" -mapreducedim - """ seekstart(s) @@ -6415,15 +6329,6 @@ the group owning the file """ operm -""" - cumsum(A, [dim]) - -Cumulative sum along a dimension `dim` (defaults to 1). See also [`cumsum!`](:func:`cumsum!`) -to use a preallocated output array, both for performance and to control the precision of the -output (e.g. to avoid overflow). -""" -cumsum - """ rpad(string, n, p) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index f4f4a676d2112..385e9e218f891 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -496,8 +496,56 @@ for (f, fmod, op) = ((:cummin, :_cummin!, :min), (:cummax, :_cummax!, :max)) @eval ($f)(A::AbstractArray) = ($f)(A, 1) end - cumsum(A::AbstractArray, axis::Integer=1) = cumsum!(similar(A, Base._cumsum_type(A)), A, axis) +""" + cumsum(A, dim=1) + +Cumulative sum along a dimension `dim` (defaults to 1). See also [`cumsum!`](:func:`cumsum!`) +to use a preallocated output array, both for performance and to control the precision of the +output (e.g. to avoid overflow). + +```jldoctest +julia> a = [1 2 3; 4 5 6] +2×3 Array{Int64,2}: + 1 2 3 + 4 5 6 + +julia> cumsum(a,1) +2×3 Array{Int64,2}: + 1 2 3 + 5 7 9 + +julia> cumsum(a,2) +2×3 Array{Int64,2}: + 1 3 6 + 4 9 15 +``` +""" +cumsum(A::AbstractArray, axis::Integer=1) = cumsum!(similar(A, Base._cumsum_type(A)), A, axis) cumsum!(B, A::AbstractArray) = cumsum!(B, A, 1) +""" + cumprod(A, dim=1) + +Cumulative product along a dimension `dim` (defaults to 1). See also +[`cumprod!`](:func:`cumprod!`) to use a preallocated output array, both for performance and +to control the precision of the output (e.g. to avoid overflow). + +```jldoctest +julia> a = [1 2 3; 4 5 6] +2×3 Array{Int64,2}: + 1 2 3 + 4 5 6 + +julia> cumprod(a,1) +2×3 Array{Int64,2}: + 1 2 3 + 4 10 18 + +julia> cumprod(a,2) +2×3 Array{Int64,2}: + 1 2 6 + 4 20 120 +``` +""" cumprod(A::AbstractArray, axis::Integer=1) = cumprod!(similar(A), A, axis) cumprod!(B, A) = cumprod!(B, A, 1) diff --git a/base/reducedim.jl b/base/reducedim.jl index f6fa07169a511..047cc9af80e92 100644 --- a/base/reducedim.jl +++ b/base/reducedim.jl @@ -225,11 +225,65 @@ mapreducedim!(f, op, R::AbstractArray, A::AbstractArray) = reducedim!{RT}(op, R::AbstractArray{RT}, A::AbstractArray) = mapreducedim!(identity, op, R, A, zero(RT)) +""" + mapreducedim(f, op, A, region[, v0]) + +Evaluates to the same as `reducedim(op, map(f, A), region, f(v0))`, but is generally +faster because the intermediate array is avoided. + +```jldoctest +julia> a = reshape(collect(1:16), (4,4)) +4×4 Array{Int64,2}: + 1 5 9 13 + 2 6 10 14 + 3 7 11 15 + 4 8 12 16 + +julia> mapreducedim(isodd, *, a, 1) +1×4 Array{Bool,2}: + false false false false + +julia> mapreducedim(isodd, |, a, 1, true) +1×4 Array{Bool,2}: + true true true true +``` +""" mapreducedim(f, op, A::AbstractArray, region, v0) = mapreducedim!(f, op, reducedim_initarray(A, region, v0), A) mapreducedim{T}(f, op, A::AbstractArray{T}, region) = mapreducedim!(f, op, reducedim_init(f, op, A, region), A) +""" + reducedim(f, A, region[, v0]) + +Reduce 2-argument function `f` along dimensions of `A`. `region` is a vector specifying the +dimensions to reduce, and `v0` is the initial value to use in the reductions. For `+`, `*`, +`max` and `min` the `v0` argument is optional. + +The associativity of the reduction is implementation-dependent; if you need a particular +associativity, e.g. left-to-right, you should write your own loop. See documentation for +[`reduce`](:func:`reduce`). + +```jldoctest +julia> a = reshape(collect(1:16), (4,4)) +4×4 Array{Int64,2}: + 1 5 9 13 + 2 6 10 14 + 3 7 11 15 + 4 8 12 16 + +julia> reducedim(max, a, 2) +4×1 Array{Int64,2}: + 13 + 14 + 15 + 16 + +julia> reducedim(max, a, 1) +1×4 Array{Int64,2}: + 4 8 12 16 +``` +""" reducedim(op, A::AbstractArray, region, v0) = mapreducedim(identity, op, A, region, v0) reducedim(op, A::AbstractArray, region) = mapreducedim(identity, op, A, region) diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index 448dd55c0caa9..f72f67f72b10a 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -657,24 +657,58 @@ Indexing, Assignment, and Concatenation Array functions --------------- -.. function:: cumprod(A, [dim]) +.. function:: cumprod(A, dim=1) .. Docstring generated from Julia source Cumulative product along a dimension ``dim`` (defaults to 1). See also :func:`cumprod!` to use a preallocated output array, both for performance and to control the precision of the output (e.g. to avoid overflow). + .. doctest:: + + julia> a = [1 2 3; 4 5 6] + 2×3 Array{Int64,2}: + 1 2 3 + 4 5 6 + + julia> cumprod(a,1) + 2×3 Array{Int64,2}: + 1 2 3 + 4 10 18 + + julia> cumprod(a,2) + 2×3 Array{Int64,2}: + 1 2 6 + 4 20 120 + .. function:: cumprod!(B, A, [dim]) .. Docstring generated from Julia source Cumulative product of ``A`` along a dimension, storing the result in ``B``\ . The dimension defaults to 1. -.. function:: cumsum(A, [dim]) +.. function:: cumsum(A, dim=1) .. Docstring generated from Julia source Cumulative sum along a dimension ``dim`` (defaults to 1). See also :func:`cumsum!` to use a preallocated output array, both for performance and to control the precision of the output (e.g. to avoid overflow). + .. doctest:: + + julia> a = [1 2 3; 4 5 6] + 2×3 Array{Int64,2}: + 1 2 3 + 4 5 6 + + julia> cumsum(a,1) + 2×3 Array{Int64,2}: + 1 2 3 + 5 7 9 + + julia> cumsum(a,2) + 2×3 Array{Int64,2}: + 1 3 6 + 4 9 15 + .. function:: cumsum!(B, A, [dim]) .. Docstring generated from Julia source @@ -717,49 +751,193 @@ Array functions Rotate matrix ``A`` 180 degrees. + .. doctest:: + + julia> a = [1 2; 3 4] + 2×2 Array{Int64,2}: + 1 2 + 3 4 + + julia> rot180(a) + 2×2 Array{Int64,2}: + 4 3 + 2 1 + .. function:: rot180(A, k) .. Docstring generated from Julia source Rotate matrix ``A`` 180 degrees an integer ``k`` number of times. If ``k`` is even, this is equivalent to a ``copy``\ . + .. doctest:: + + julia> a = [1 2; 3 4] + 2×2 Array{Int64,2}: + 1 2 + 3 4 + + julia> rot180(a,1) + 2×2 Array{Int64,2}: + 4 3 + 2 1 + + julia> rot180(a,2) + 2×2 Array{Int64,2}: + 1 2 + 3 4 + .. function:: rotl90(A) .. Docstring generated from Julia source Rotate matrix ``A`` left 90 degrees. + .. doctest:: + + julia> a = [1 2; 3 4] + 2×2 Array{Int64,2}: + 1 2 + 3 4 + + julia> rotl90(a) + 2×2 Array{Int64,2}: + 2 4 + 1 3 + .. function:: rotl90(A, k) .. Docstring generated from Julia source Rotate matrix ``A`` left 90 degrees an integer ``k`` number of times. If ``k`` is zero or a multiple of four, this is equivalent to a ``copy``\ . + .. doctest:: + + julia> a = [1 2; 3 4] + 2×2 Array{Int64,2}: + 1 2 + 3 4 + + julia> rotl90(a,1) + 2×2 Array{Int64,2}: + 2 4 + 1 3 + + julia> rotl90(a,2) + 2×2 Array{Int64,2}: + 4 3 + 2 1 + + julia> rotl90(a,3) + 2×2 Array{Int64,2}: + 3 1 + 4 2 + + julia> rotl90(a,4) + 2×2 Array{Int64,2}: + 1 2 + 3 4 + .. function:: rotr90(A) .. Docstring generated from Julia source Rotate matrix ``A`` right 90 degrees. + .. doctest:: + + julia> a = [1 2; 3 4] + 2×2 Array{Int64,2}: + 1 2 + 3 4 + + julia> rotr90(a) + 2×2 Array{Int64,2}: + 3 1 + 4 2 + .. function:: rotr90(A, k) .. Docstring generated from Julia source Rotate matrix ``A`` right 90 degrees an integer ``k`` number of times. If ``k`` is zero or a multiple of four, this is equivalent to a ``copy``\ . -.. function:: reducedim(f, A, dims[, initial]) + .. doctest:: + + julia> a = [1 2; 3 4] + 2×2 Array{Int64,2}: + 1 2 + 3 4 + + julia> rotr90(a,1) + 2×2 Array{Int64,2}: + 3 1 + 4 2 + + julia> rotr90(a,2) + 2×2 Array{Int64,2}: + 4 3 + 2 1 + + julia> rotr90(a,3) + 2×2 Array{Int64,2}: + 2 4 + 1 3 + + julia> rotr90(a,4) + 2×2 Array{Int64,2}: + 1 2 + 3 4 + +.. function:: reducedim(f, A, region[, v0]) .. Docstring generated from Julia source - Reduce 2-argument function ``f`` along dimensions of ``A``\ . ``dims`` is a vector specifying the dimensions to reduce, and ``initial`` is the initial value to use in the reductions. For ``+``\ , ``*``\ , ``max`` and ``min`` the ``initial`` argument is optional. + Reduce 2-argument function ``f`` along dimensions of ``A``\ . ``region`` is a vector specifying the dimensions to reduce, and ``v0`` is the initial value to use in the reductions. For ``+``\ , ``*``\ , ``max`` and ``min`` the ``v0`` argument is optional. + + The associativity of the reduction is implementation-dependent; if you need a particular associativity, e.g. left-to-right, you should write your own loop. See documentation for :func:`reduce`\ . + + .. doctest:: - The associativity of the reduction is implementation-dependent; if you need a particular associativity, e.g. left-to-right, you should write your own loop. See documentation for ``reduce``\ . + julia> a = reshape(collect(1:16), (4,4)) + 4×4 Array{Int64,2}: + 1 5 9 13 + 2 6 10 14 + 3 7 11 15 + 4 8 12 16 + + julia> reducedim(max, a, 2) + 4×1 Array{Int64,2}: + 13 + 14 + 15 + 16 + + julia> reducedim(max, a, 1) + 1×4 Array{Int64,2}: + 4 8 12 16 -.. function:: mapreducedim(f, op, A, dims[, initial]) +.. function:: mapreducedim(f, op, A, region[, v0]) .. Docstring generated from Julia source - Evaluates to the same as ``reducedim(op, map(f, A), dims, f(initial))``\ , but is generally faster because the intermediate array is avoided. + Evaluates to the same as ``reducedim(op, map(f, A), region, f(v0))``\ , but is generally faster because the intermediate array is avoided. + + .. doctest:: + + julia> a = reshape(collect(1:16), (4,4)) + 4×4 Array{Int64,2}: + 1 5 9 13 + 2 6 10 14 + 3 7 11 15 + 4 8 12 16 + + julia> mapreducedim(isodd, *, a, 1) + 1×4 Array{Bool,2}: + false false false false + + julia> mapreducedim(isodd, |, a, 1, true) + 1×4 Array{Bool,2}: + true true true true .. function:: mapslices(f, A, dims) @@ -767,6 +945,40 @@ Array functions Transform the given dimensions of array ``A`` using function ``f``\ . ``f`` is called on each slice of ``A`` of the form ``A[...,:,...,:,...]``\ . ``dims`` is an integer vector specifying where the colons go in this expression. The results are concatenated along the remaining dimensions. For example, if ``dims`` is ``[1,2]`` and ``A`` is 4-dimensional, ``f`` is called on ``A[:,:,i,j]`` for all ``i`` and ``j``\ . + .. doctest:: + + julia> a = reshape(collect(1:16),(2,2,2,2)) + 2×2×2×2 Array{Int64,4}: + [:, :, 1, 1] = + 1 3 + 2 4 + + [:, :, 2, 1] = + 5 7 + 6 8 + + [:, :, 1, 2] = + 9 11 + 10 12 + + [:, :, 2, 2] = + 13 15 + 14 16 + + julia> mapslices(sum, a, [1,2]) + 1×1×2×2 Array{Int64,4}: + [:, :, 1, 1] = + 10 + + [:, :, 2, 1] = + 26 + + [:, :, 1, 2] = + 42 + + [:, :, 2, 2] = + 58 + .. function:: sum_kbn(A) .. Docstring generated from Julia source From 0e7933d0f51c216a0bbb155fcc0583e419e38cfe Mon Sep 17 00:00:00 2001 From: Katharine Hyatt Date: Fri, 5 Aug 2016 08:27:20 -0700 Subject: [PATCH 07/80] Added documentation for doctests (#17825) The manual entry for doctests was way out of date! Updated with an example of what to do. (cherry picked from commit 6d3e3378649fbad886f3fa49ea11b09a0917f075) --- doc/manual/documentation.rst | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/doc/manual/documentation.rst b/doc/manual/documentation.rst index f03f60a8b4ba2..a6d481f340a58 100644 --- a/doc/manual/documentation.rst +++ b/doc/manual/documentation.rst @@ -99,14 +99,29 @@ documentation: ... """ -5. Group examples under an ``# Examples`` section and use ````julia`` blocks instead of +5. Group examples under an ``# Examples`` section and use `````julia`` blocks instead of standard text. Examples should consist of verbatim copies of the Julia REPL, including the ``julia>`` - prompt (see example above). This will be used in the future to allow running examples - automatically and checking that their actual output is consistent with that presented - in the documentation (a feature called *doctests*). This way, the code will be tested and - examples won't get out of date without notice. + prompt (see example above). This is used to allow running examples automatically and + checking that their actual output is consistent with that presented in the + documentation (a feature called *doctests*). This way, the code will be tested and + examples won't get out of date without notice. An example:: + + """ + Some nice documentation here. + + ```jldoctest + julia> a = [1 2; 3 4] + 2×2 Array{Int64,2}: + 1 2 + 3 4 + ``` + """ + + You can then run ``make -C doc doctest`` to run all the doctests, which will ensure + that your example works. Note that whitespace in your doctest is significant! The + doctest will fail if you misalign the output of pretty-printing an array, for example. 6. Use backticks to identify code and equations. From aee17b6dcfbc6ae2544160ee7bc537be26228273 Mon Sep 17 00:00:00 2001 From: Katharine Hyatt Date: Fri, 5 Aug 2016 08:28:09 -0700 Subject: [PATCH 08/80] Move math docs out of HelpDB, more examples, fix typos (#17791) * Move math docs out of HelpDB, more examples, fix typos Found a series of typos in `cov` and friends. Added more notes about `NaN` and Julia. Made the function signatures reflect what's actually in the code. More examples for quite a few functions. * Move quadgk docs out, update formatting * Moved special functions out of HelpDB, insert some links * Updated docs for some array ops as well * Updated in response to feedback Removed calls to `rand` in doctests. Made examples better. Cleaned up function signatures. (cherry picked from commit 2d24eda52727c6cbccec935d689d84d710366f5a) --- base/abstractarray.jl | 113 ++++++- base/abstractarraymath.jl | 89 ++++++ base/array.jl | 5 +- base/docs/helpdb/Base.jl | 335 -------------------- base/iterator.jl | 12 +- base/math.jl | 16 + base/quadgk.jl | 55 ++++ base/rational.jl | 22 +- base/reduce.jl | 2 +- base/special/bessel.jl | 104 +++++- base/statistics.jl | 115 ++++++- doc/manual/arrays.rst | 2 +- doc/manual/complex-and-rational-numbers.rst | 2 +- doc/manual/control-flow.rst | 2 +- doc/manual/strings.rst | 2 +- doc/stdlib/arrays.rst | 136 +++++++- doc/stdlib/collections.rst | 21 +- doc/stdlib/math.rst | 116 +++++-- doc/stdlib/pkg.rst | 2 +- 19 files changed, 751 insertions(+), 400 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 08d5956596f17..7a6ead53c946a 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1017,7 +1017,80 @@ function cat_t(catdims, typeC::Type, X...) return C end +""" + vcat(A...) + +Concatenate along dimension 1. + +```jldoctest +julia> a = [1 2 3 4 5] +1×5 Array{Int64,2}: + 1 2 3 4 5 + +julia> b = [6 7 8 9 10; 11 12 13 14 15] +2×5 Array{Int64,2}: + 6 7 8 9 10 + 11 12 13 14 15 + +julia> vcat(a,b) +3×5 Array{Int64,2}: + 1 2 3 4 5 + 6 7 8 9 10 + 11 12 13 14 15 + +julia> c = ([1 2 3], [4 5 6]) +( +[1 2 3], + +[4 5 6]) + +julia> vcat(c...) +2×3 Array{Int64,2}: + 1 2 3 + 4 5 6 +``` +""" vcat(X...) = cat(1, X...) +""" + hcat(A...) + +Concatenate along dimension 2. + +```jldoctest +julia> a = [1; 2; 3; 4; 5] +5-element Array{Int64,1}: + 1 + 2 + 3 + 4 + 5 + +julia> b = [6 7; 8 9; 10 11; 12 13; 14 15] +5×2 Array{Int64,2}: + 6 7 + 8 9 + 10 11 + 12 13 + 14 15 + +julia> hcat(a,b) +5×3 Array{Int64,2}: + 1 6 7 + 2 8 9 + 3 10 11 + 4 12 13 + 5 14 15 + +julia> c = ([1; 2; 3], [4; 5; 6]) +([1,2,3],[4,5,6]) + +julia> hcat(c...) +3×2 Array{Int64,2}: + 1 4 + 2 5 + 3 6 +``` +""" hcat(X...) = cat(2, X...) typed_vcat(T::Type, X...) = cat_t(1, T, X...) @@ -1054,6 +1127,43 @@ function hvcat(nbc::Integer, as...) hvcat(ntuple(i->nbc, nbr), as...) end +""" + hvcat(rows::Tuple{Vararg{Int}}, values...) + +Horizontal and vertical concatenation in one call. This function is called for block matrix +syntax. The first argument specifies the number of arguments to concatenate in each block +row. + +```jldoctest +julia> a, b, c, d, e, f = 1, 2, 3, 4, 5, 6 +(1,2,3,4,5,6) + +julia> [a b c; d e f] +2×3 Array{Int64,2}: + 1 2 3 + 4 5 6 + +julia> hvcat((3,3), a,b,c,d,e,f) +2×3 Array{Int64,2}: + 1 2 3 + 4 5 6 + +julia> [a b;c d; e f] +3×2 Array{Int64,2}: + 1 2 + 3 4 + 5 6 + +julia> hvcat((2,2,2), a,b,c,d,e,f) +3×2 Array{Int64,2}: + 1 2 + 3 4 + 5 6 +``` + +If the first argument is a single integer `n`, then all block rows are assumed to have `n` +block columns. +""" hvcat(rows::Tuple{Vararg{Int}}, xs::AbstractMatrix...) = typed_hvcat(promote_eltype(xs...), rows, xs...) hvcat{T}(rows::Tuple{Vararg{Int}}, xs::AbstractMatrix{T}...) = typed_hvcat(T, rows, xs...) @@ -1340,8 +1450,7 @@ For multiple iterable arguments, `f` is called elementwise. needed, for example in `foreach(println, array)`. ```jldoctest -julia> a -1:3:7 +julia> a = 1:3:7; julia> foreach(x->println(x^2),a) 1 diff --git a/base/abstractarraymath.jl b/base/abstractarraymath.jl index 61b95287155ad..7f95b1f1651ea 100644 --- a/base/abstractarraymath.jl +++ b/base/abstractarraymath.jl @@ -11,6 +11,27 @@ transpose(a::AbstractArray) = error("transpose not implemented for $(typeof(a)). ## Constructors ## +""" + vec(a::AbstractArray) -> Vector + +Reshape array `a` as a one-dimensional column vector. + +```jldoctest +julia> a = [1 2 3; 4 5 6] +2×3 Array{Int64,2}: + 1 2 3 + 4 5 6 + +julia> vec(a) +6-element Array{Int64,1}: + 1 + 4 + 2 + 5 + 3 + 6 +``` +""" vec(a::AbstractArray) = reshape(a,_length(a)) vec(a::AbstractVector) = a @@ -18,6 +39,27 @@ _sub(::Tuple{}, ::Tuple{}) = () _sub(t::Tuple, ::Tuple{}) = t _sub(t::Tuple, s::Tuple) = _sub(tail(t), tail(s)) +""" + squeeze(A, dims) + +Remove the dimensions specified by `dims` from array `A`. +Elements of `dims` must be unique and within the range `1:ndims(A)`. +`size(A,i)` must equal 1 for all `i` in `dims`. + +```jldoctest +julia> a = reshape(collect(1:4),(2,2,1,1)) +2×2×1×1 Array{Int64,4}: +[:, :, 1, 1] = + 1 3 + 2 4 + +julia> squeeze(a,3) +2×2×1 Array{Int64,3}: +[:, :, 1] = + 1 3 + 2 4 +``` +""" function squeeze(A::AbstractArray, dims::Dims) for i in 1:length(dims) 1 <= dims[i] <= ndims(A) || throw(ArgumentError("squeezed dims must be in range 1:ndims(A)")) @@ -71,6 +113,23 @@ function flipdim(A::AbstractVector, d::Integer) reverse(A) end +""" + flipdim(A, d) + +Reverse `A` in dimension `d`. + +```jldoctest +julia> b = [1 2; 3 4] +2×2 Array{Int64,2}: + 1 2 + 3 4 + +julia> flipdim(b,2) +2×2 Array{Int64,2}: + 2 1 + 4 3 +``` +""" function flipdim(A::AbstractArray, d::Integer) nd = ndims(A) 1 ≤ d ≤ nd || throw(ArgumentError("dimension $d is not 1 ≤ $d ≤ $nd")) @@ -100,6 +159,36 @@ function flipdim(A::AbstractArray, d::Integer) end circshift(a::AbstractArray, shiftamt::Real) = circshift(a, [Integer(shiftamt)]) + +""" + circshift(A, shifts) + +Circularly shift the data in an array. The second argument is a vector giving the amount to +shift in each dimension. + +```jldoctest +julia> b = reshape(collect(1:16), (4,4)) +4×4 Array{Int64,2}: + 1 5 9 13 + 2 6 10 14 + 3 7 11 15 + 4 8 12 16 + +julia> circshift(b, [0,2]) +4×4 Array{Int64,2}: + 9 13 1 5 + 10 14 2 6 + 11 15 3 7 + 12 16 4 8 + +julia> circshift(b, [-1,0]) +4×4 Array{Int64,2}: + 2 6 10 14 + 3 7 11 15 + 4 8 12 16 + 1 5 9 13 +``` +""" function circshift{T,N}(a::AbstractArray{T,N}, shiftamts) I = () for i=1:N diff --git a/base/array.jl b/base/array.jl index 079bdfd1e5d99..f892f2f6689d7 100644 --- a/base/array.jl +++ b/base/array.jl @@ -667,6 +667,7 @@ function hcat{T}(V::Vector{T}...) end return [ V[j][i]::T for i=1:length(V[1]), j=1:length(V) ] end + function vcat{T}(arrays::Vector{T}...) n = 0 for a in arrays @@ -860,7 +861,7 @@ Returns the minimum element and its index. The collection must not be empty. ```jldoctest -julia> findmax([8,0.1,-9,pi]) +julia> findmin([8,0.1,-9,pi]) (-9.0,3) ``` """ @@ -915,7 +916,7 @@ vector contains 0 wherever `a` is not a member of `b`. ```jldoctest julia> a = ['a', 'b', 'c', 'b', 'd', 'a']; -julia> b = ['a','b','c'] +julia> b = ['a','b','c']; julia> indexin(a,b) 6-element Array{Int64,1}: diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index b8ab785fd9778..9a08432feec8e 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -146,13 +146,6 @@ Print (using [`print`](:func:`print`)) `x` followed by a newline. """ println -""" - besselj(nu, x) - -Bessel function of the first kind of order `nu`, ``J_\\nu(x)``. -""" -besselj - """ //(num, den) @@ -211,13 +204,6 @@ Multiply elements of `A` over the singleton dimensions of `r`, and write results """ prod! -""" - airybi(x) - -Airy function ``\\operatorname{Bi}(x)``. -""" -airybi - """ gensym([tag]) @@ -356,13 +342,6 @@ cannot be used with empty collections (see `reduce(op, itr)`). """ foldl(op, itr) -""" - airybiprime(x) - -Airy function derivative ``\\operatorname{Bi}'(x)``. -""" -airybiprime - """ Ac_rdiv_B(A, B) @@ -757,13 +736,6 @@ Get the local machine's host name. """ gethostname -""" - hankelh1x(nu, x) - -Scaled Bessel function of the third kind of order `nu`, ``H^{(1)}_\\nu(x) e^{-x i}``. -""" -hankelh1x - """ replace(string, pat, r[, n]) @@ -913,13 +885,6 @@ This would create a 25-by-30000 `BitArray`, linked to the file associated with s """ Mmap.mmap(io, ::BitArray, dims = ?, offset = ?) -""" - airyprime(x) - -Airy function derivative ``\\operatorname{Ai}'(x)``. -""" -airyprime - """ bessely0(x) @@ -950,13 +915,6 @@ Decodes the base64-encoded `string` and returns a `Vector{UInt8}` of the decoded """ base64decode -""" - besselkx(nu, x) - -Scaled modified Bessel function of the second kind of order `nu`, ``K_\\nu(x) e^x``. -""" -besselkx - """ oct(n, [pad]) @@ -1213,13 +1171,6 @@ Wrap an expression in a `Task` and add it to the local machine's scheduler queue """ :@schedule -""" - bessely(nu, x) - -Bessel function of the second kind of order `nu`, ``Y_\\nu(x)``. -""" -bessely - """ gradient(F, [h]) @@ -1389,15 +1340,6 @@ Connect to the named pipe / UNIX domain socket at `path`. """ connect(path) -""" - mean(v[, region]) - -Compute the mean of whole array `v`, or optionally along the dimensions in `region`. Note: -Julia does not ignore `NaN` values in the computation. For applications requiring the -handling of missing data, the `DataArray` package is recommended. -""" -mean - """ split(string, [chars]; limit=0, keep=true) @@ -1440,13 +1382,6 @@ be passed, to be returned from the last `produce` call in the producer. """ consume -""" - hankelh2x(nu, x) - -Scaled Bessel function of the third kind of order `nu`, ``H^{(2)}_\\nu(x) e^{x i}``. -""" -hankelh2x - """ ndigits(n, b = 10) @@ -1594,14 +1529,6 @@ Dict{String,Float64} with 3 entries: """ merge -""" - circshift(A,shifts) - -Circularly shift the data in an array. The second argument is a vector giving the amount to -shift in each dimension. -""" -circshift - """ yield() @@ -1720,13 +1647,6 @@ Construct a tuple of the given objects. """ tuple -""" - besseli(nu, x) - -Modified Bessel function of the first kind of order `nu`, ``I_\\nu(x)``. -""" -besseli - """ eachmatch(r::Regex, s::AbstractString[, overlap::Bool=false]) @@ -1848,14 +1768,6 @@ Bitwise and. """ & -""" - besselyx(nu, x) - -Scaled Bessel function of the second kind of order `nu`, -``Y_\\nu(x) e^{- | \\operatorname{Im}(x) |}``. -""" -besselyx - """ eigmax(A) @@ -1910,13 +1822,6 @@ and/or speed. """ logdet -""" - hcat(A...) - -Concatenate along dimension 2. -""" -hcat - """ select(v, k, [by=,] [lt=,] [rev=false]) @@ -2130,14 +2035,6 @@ Element-wise greater-than-or-equals comparison operator. """ Base.:(.>=) -""" - stdm(v, m) - -Compute the sample standard deviation of a vector `v` with known mean `m`. Note: Julia does -not ignore `NaN` values in the computation. -""" -stdm - """ mv(src::AbstractString,dst::AbstractString; remove_destination::Bool=false) @@ -2217,13 +2114,6 @@ In-place version of [`reverse`](:func:`reverse`). """ reverse! -""" - flipdim(A, d) - -Reverse `A` in dimension `d`. -""" -flipdim - """ num(x) @@ -2297,13 +2187,6 @@ Bessel function of the second kind of order 1, ``Y_1(x)``. """ bessely1 -""" - besseljx(nu, x) - -Scaled Bessel function of the first kind of order `nu`, ``J_\\nu(x) e^{- | \\operatorname{Im}(x) |}``. -""" -besseljx - """ print(x) @@ -2530,45 +2413,6 @@ Extract a named field from a `value` of composite type. The syntax `a.b` calls """ getfield -""" - hvcat(rows::Tuple{Vararg{Int}}, values...) - -Horizontal and vertical concatenation in one call. This function is called for block matrix -syntax. The first argument specifies the number of arguments to concatenate in each block -row. - -```jldoctest -julia> a, b, c, d, e, f = 1, 2, 3, 4, 5, 6 -(1,2,3,4,5,6) - -julia> [a b c; d e f] -2×3 Array{Int64,2}: - 1 2 3 - 4 5 6 - -julia> hvcat((3,3), a,b,c,d,e,f) -2×3 Array{Int64,2}: - 1 2 3 - 4 5 6 - -julia> [a b;c d; e f] -3×2 Array{Int64,2}: - 1 2 - 3 4 - 5 6 - -julia> hvcat((2,2,2), a,b,c,d,e,f) -3×2 Array{Int64,2}: - 1 2 - 3 4 - 5 6 -``` - -If the first argument is a single integer `n`, then all block rows are assumed to have `n` -block columns. -""" -hvcat - """ besselj1(x) @@ -2700,13 +2544,6 @@ Create all directories in the given `path`, with permissions `mode`. `mode` defa """ mkpath -""" - besselix(nu, x) - -Scaled modified Bessel function of the first kind of order `nu`, ``I_\\nu(x) e^{- | \\operatorname{Re}(x) |}``. -""" -besselix - """ union(s1,s2...) ∪(s1,s2...) @@ -3183,17 +3020,6 @@ Compute ``\\sin(\\pi x) / (\\pi x)`` if ``x \\neq 0``, and ``1`` if ``x = 0``. """ sinc -""" - median(v[, region]) - -Compute the median of whole array `v`, or optionally along the dimensions in `region`. For -even number of elements no exact median element exists, so the result is equivalent to -calculating mean of two median elements. `NaN` is returned if the data contains any `NaN` -values. For applications requiring the handling of missing data, the `DataArrays` package is -recommended. -""" -median - """ cglobal((symbol, library) [, type=Void]) @@ -3212,14 +3038,6 @@ itself). For matrices, returns an identity matrix of the appropriate size and ty """ one -""" - rationalize([Type=Int,] x; tol=eps(x)) - -Approximate floating point number `x` as a Rational number with components of the given -integer type. The result will differ from `x` by no more than `tol`. -""" -rationalize - """ splice!(collection, index, [replacement]) -> item @@ -3437,13 +3255,6 @@ Test whether `n` is a power of two. """ ispow2 -""" - vcat(A...) - -Concatenate along dimension 1. -""" -vcat - """ isgraph(c::Union{Char,AbstractString}) -> Bool @@ -3805,14 +3616,6 @@ array, but with the specified element type. """ reinterpret -""" - squeeze(A, dims) - -Remove the dimensions specified by `dims` from array `A`. Elements of `dims` must be unique -and within the range `1:ndims(A)`. -""" -squeeze - """ ~(x) @@ -3820,13 +3623,6 @@ Bitwise not. """ ~ -""" - hankelh1(nu, x) - -Bessel function of the third kind of order `nu`, ``H^{(1)}_\\nu(x)``. -""" -hankelh1 - """ rem(x, y) %(x, y) @@ -4141,18 +3937,6 @@ Delete the mapping for the given key in a collection, and return the collection. """ delete! -""" - std(v[, region]) - -Compute the sample standard deviation of a vector or array `v`, optionally along dimensions -in `region`. The algorithm returns an estimator of the generative distribution's standard -deviation under the assumption that each entry of `v` is an IID drawn from that generative -distribution. This computation is equivalent to calculating `sqrt(sum((v - mean(v)).^2) / -(length(v) - 1))`. Note: Julia does not ignore `NaN` values in the computation. For -applications requiring the handling of missing data, the `DataArray` package is recommended. -""" -std - """ chr2ind(string, i) @@ -4275,13 +4059,6 @@ Get a module's enclosing `Module`. `Main` is its own parent, as is `LastMain` af """ module_parent -""" - airyaiprime(x) - -Airy function derivative ``\\operatorname{Ai}'(x)``. -""" -airyaiprime - """ prepend!(collection, items) -> collection @@ -5043,13 +4820,6 @@ called in last in first out (LIFO) order and run before object finalizers. """ atexit -""" - besselk(nu, x) - -Modified Bessel function of the second kind of order `nu`, ``K_\\nu(x)``. -""" -besselk - """ readchomp(x) @@ -5653,13 +5423,6 @@ general. """ setdiff -""" - airyai(x) - -Airy function ``\\operatorname{Ai}(x)``. -""" -airyai - """ error(message::AbstractString) @@ -5982,14 +5745,6 @@ Compute the secant of `x`, where `x` is in degrees. """ secd -""" - varm(v, m) - -Compute the sample variance of a vector `v` with known mean `m`. Note: Julia does not ignore -`NaN` values in the computation. -""" -varm - """ OverflowError() @@ -6113,13 +5868,6 @@ julia> leading_zeros(Int32(1)) """ leading_zeros -""" - hankelh2(nu, x) - -Bessel function of the third kind of order `nu`, ``H^{(2)}_\\nu(x)``. -""" -hankelh2 - """ lexcmp(x, y) @@ -6781,13 +6529,6 @@ characters, tests whether the last character of `string` belongs to that set. """ endswith -""" - airy(k,x) - -The `k`th derivative of the Airy function ``\\operatorname{Ai}(x)``. -""" -airy - """ !(x) @@ -7192,15 +6933,6 @@ Get the backtrace of the current exception, for use within `catch` blocks. """ catch_backtrace -""" - airyx(k,x) - -scaled `k`th derivative of the Airy function, return ``\\operatorname{Ai}(x) e^{\\frac{2}{3} x \\sqrt{x}}`` -for `k == 0 || k == 1`, and ``\\operatorname{Ai}(x) e^{- \\left| \\operatorname{Re} \\left( \\frac{2}{3} x \\sqrt{x} \\right) \\right|}`` -for `k == 2 || k == 3`. -""" -airyx - """ get_zero_subnormals() -> Bool @@ -7553,13 +7285,6 @@ available in the `.error` field. """ InitError -""" - vec(Array) -> Vector - -Vectorize an array using column-major convention. -""" -vec - """ copy!(dest, src) @@ -8159,58 +7884,6 @@ Compute ``x \\times 2^n``. """ ldexp -""" - quadgk(f, a,b,c...; reltol=sqrt(eps), abstol=0, maxevals=10^7, order=7, norm=vecnorm) - -Numerically integrate the function `f(x)` from `a` to `b`, and optionally over additional -intervals `b` to `c` and so on. Keyword options include a relative error tolerance `reltol` -(defaults to `sqrt(eps)` in the precision of the endpoints), an absolute error tolerance -`abstol` (defaults to 0), a maximum number of function evaluations `maxevals` (defaults to -`10^7`), and the `order` of the integration rule (defaults to 7). - -Returns a pair `(I,E)` of the estimated integral `I` and an estimated upper bound on the -absolute error `E`. If `maxevals` is not exceeded then `E <= max(abstol, reltol*norm(I))` -will hold. (Note that it is useful to specify a positive `abstol` in cases where `norm(I)` -may be zero.) - -The endpoints `a` etcetera can also be complex (in which case the integral is performed over -straight-line segments in the complex plane). If the endpoints are `BigFloat`, then the -integration will be performed in `BigFloat` precision as well (note: it is advisable to -increase the integration `order` in rough proportion to the precision, for smooth -integrands). More generally, the precision is set by the precision of the integration -endpoints (promoted to floating-point types). - -The integrand `f(x)` can return any numeric scalar, vector, or matrix type, or in fact any -type supporting `+`, `-`, multiplication by real values, and a `norm` (i.e., any normed -vector space). Alternatively, a different norm can be specified by passing a `norm`-like -function as the `norm` keyword argument (which defaults to `vecnorm`). - -\[Only one-dimensional integrals are provided by this function. For multi-dimensional -integration (cubature), there are many different algorithms (often much better than simple -nested 1d integrals) and the optimal choice tends to be very problem-dependent. See the -Julia external-package listing for available algorithms for multidimensional integration or -other specialized tasks (such as integrals of highly oscillatory or singular functions).\] - -The algorithm is an adaptive Gauss-Kronrod integration technique: the integral in each -interval is estimated using a Kronrod rule (`2*order+1` points) and the error is estimated -using an embedded Gauss rule (`order` points). The interval with the largest error is then -subdivided into two intervals and the process is repeated until the desired error tolerance -is achieved. - -These quadrature rules work best for smooth functions within each interval, so if your -function has a known discontinuity or other singularity, it is best to subdivide your -interval to put the singularity at an endpoint. For example, if `f` has a discontinuity at -`x=0.7` and you want to integrate from 0 to 1, you should use `quadgk(f, 0,0.7,1)` to -subdivide the interval at the point of discontinuity. The integrand is never evaluated -exactly at the endpoints of the intervals, so it is possible to integrate functions that -diverge at the endpoints as long as the singularity is integrable (for example, a `log(x)` -or `1/sqrt(x)` singularity). - -For real-valued endpoints, the starting and/or ending points may be infinite. (A coordinate -transformation is performed internally to map the infinite interval to a finite one.) -""" -quadgk - """ islower(c::Union{Char,AbstractString}) -> Bool @@ -8268,14 +7941,6 @@ Returns `true` if the value of the sign of `x` is negative, otherwise `false`. """ signbit -""" - clamp(x, lo, hi) - -Return `x` if `lo <= x <= hi`. If `x < lo`, return `lo`. If `x > hi`, return `hi`. Arguments -are promoted to a common type. Operates elementwise over `x` if it is an array. -""" -clamp - """ cscd(x) diff --git a/base/iterator.jl b/base/iterator.jl index 6c93bcbb6be37..1e11264915a91 100644 --- a/base/iterator.jl +++ b/base/iterator.jl @@ -292,13 +292,13 @@ julia> collect(a) 5 7 9 - 11 + 11 julia> collect(take(a,3)) 3-element Array{Int64,1}: - 1 - 3 - 5 + 1 + 3 + 5 ``` """ take(xs, n::Int) = Take(xs, n) @@ -346,12 +346,12 @@ julia> collect(a) 5 7 9 - 11 + 11 julia> collect(drop(a,4)) 2-element Array{Int64,1}: 9 - 11 + 11 ``` """ drop(xs, n::Int) = Drop(xs, n) diff --git a/base/math.jl b/base/math.jl index 69012f7201236..6c96f5ab67319 100644 --- a/base/math.jl +++ b/base/math.jl @@ -34,6 +34,20 @@ import Core.Intrinsics: sqrt_llvm, box, unbox, powi_llvm # non-type specific math functions +""" + clamp(x, lo, hi) + +Return `x` if `lo <= x <= hi`. If `x < lo`, return `lo`. If `x > hi`, return `hi`. Arguments +are promoted to a common type. Operates elementwise over `x` if `x` is an array. + +```jldoctest +julia> clamp([pi, 1.0, big(10.)], 2., 9.) +3-element Array{BigFloat,1}: + 3.141592653589793238462643383279502884197169399375105820974944592307816406286198 + 2.000000000000000000000000000000000000000000000000000000000000000000000000000000 + 9.000000000000000000000000000000000000000000000000000000000000000000000000000000 +``` +""" clamp{X,L,H}(x::X, lo::L, hi::H) = ifelse(x > hi, convert(promote_type(X,L,H), hi), ifelse(x < lo, @@ -43,6 +57,7 @@ clamp{X,L,H}(x::X, lo::L, hi::H) = clamp{T}(x::AbstractArray{T,1}, lo, hi) = [clamp(xx, lo, hi) for xx in x] clamp{T}(x::AbstractArray{T,2}, lo, hi) = [clamp(x[i,j], lo, hi) for i in indices(x,1), j in indices(x,2)] + clamp{T}(x::AbstractArray{T}, lo, hi) = reshape([clamp(xx, lo, hi) for xx in x], size(x)) @@ -50,6 +65,7 @@ clamp{T}(x::AbstractArray{T}, lo, hi) = clamp!(array::AbstractArray, lo, hi) Restrict values in `array` to the specified range, in-place. +See also [`clamp`](:func:`clamp`). """ function clamp!{T}(x::AbstractArray{T}, lo, hi) @inbounds for i in eachindex(x) diff --git a/base/quadgk.jl b/base/quadgk.jl index dbd99d518493c..d80e1a4ae516e 100644 --- a/base/quadgk.jl +++ b/base/quadgk.jl @@ -164,6 +164,61 @@ function quadgk{T<:AbstractFloat}(f, a::Complex{T}, do_quadgk(f, [a, b, c...], order, T, abstol, reltol, maxevals, norm) end +""" + quadgk(f, a,b,c...; reltol=sqrt(eps), abstol=0, maxevals=10^7, order=7, norm=vecnorm) + +Numerically integrate the function `f(x)` from `a` to `b`, and optionally over additional +intervals `b` to `c` and so on. Keyword options include a relative error tolerance `reltol` +(defaults to `sqrt(eps)` in the precision of the endpoints), an absolute error tolerance +`abstol` (defaults to 0), a maximum number of function evaluations `maxevals` (defaults to +`10^7`), and the `order` of the integration rule (defaults to 7). + +Returns a pair `(I,E)` of the estimated integral `I` and an estimated upper bound on the +absolute error `E`. If `maxevals` is not exceeded then `E <= max(abstol, reltol*norm(I))` +will hold. (Note that it is useful to specify a positive `abstol` in cases where `norm(I)` +may be zero.) + +The endpoints `a` et cetera can also be complex (in which case the integral is performed over +straight-line segments in the complex plane). If the endpoints are `BigFloat`, then the +integration will be performed in `BigFloat` precision as well. + +!!! note + It is advisable to increase the integration `order` in rough proportion to the + precision, for smooth integrands. + +More generally, the precision is set by the precision of the integration +endpoints (promoted to floating-point types). + +The integrand `f(x)` can return any numeric scalar, vector, or matrix type, or in fact any +type supporting `+`, `-`, multiplication by real values, and a `norm` (i.e., any normed +vector space). Alternatively, a different norm can be specified by passing a `norm`-like +function as the `norm` keyword argument (which defaults to `vecnorm`). + +!!! note + Only one-dimensional integrals are provided by this function. For multi-dimensional + integration (cubature), there are many different algorithms (often much better than simple + nested 1d integrals) and the optimal choice tends to be very problem-dependent. See the + Julia external-package listing for available algorithms for multidimensional integration or + other specialized tasks (such as integrals of highly oscillatory or singular functions). + +The algorithm is an adaptive Gauss-Kronrod integration technique: the integral in each +interval is estimated using a Kronrod rule (`2*order+1` points) and the error is estimated +using an embedded Gauss rule (`order` points). The interval with the largest error is then +subdivided into two intervals and the process is repeated until the desired error tolerance +is achieved. + +These quadrature rules work best for smooth functions within each interval, so if your +function has a known discontinuity or other singularity, it is best to subdivide your +interval to put the singularity at an endpoint. For example, if `f` has a discontinuity at +`x=0.7` and you want to integrate from 0 to 1, you should use `quadgk(f, 0,0.7,1)` to +subdivide the interval at the point of discontinuity. The integrand is never evaluated +exactly at the endpoints of the intervals, so it is possible to integrate functions that +diverge at the endpoints as long as the singularity is integrable (for example, a `log(x)` +or `1/sqrt(x)` singularity). + +For real-valued endpoints, the starting and/or ending points may be infinite. (A coordinate +transformation is performed internally to map the infinite interval to a finite one.) +""" # generic version: determine precision from a combination of # all the integration-segment endpoints function quadgk(f, a, b, c...; kws...) diff --git a/base/rational.jl b/base/rational.jl index 438e8e1724f4e..067b8b570deac 100644 --- a/base/rational.jl +++ b/base/rational.jl @@ -90,8 +90,28 @@ promote_rule{T<:Integer,S<:AbstractFloat}(::Type{Rational{T}}, ::Type{S}) = prom widen{T}(::Type{Rational{T}}) = Rational{widen(T)} +""" + rationalize([T<:Integer=Int,] x; tol::Real=eps(x)) + +Approximate floating point number `x` as a `Rational` number with components +of the given integer type. The result will differ from `x` by no more than `tol`. +If `T` is not provided, it defaults to `Int`. + +```jldoctest +julia> rationalize(5.6) +28//5 + +julia> a = rationalize(BigInt, 10.3) +103//10 + +julia> typeof(num(a)) +BigInt +``` +""" function rationalize{T<:Integer}(::Type{T}, x::AbstractFloat; tol::Real=eps(x)) - tol < 0 && throw(ArgumentError("negative tolerance")) + if tol < 0 + throw(ArgumentError("negative tolerance $tol")) + end isnan(x) && return zero(T)//zero(T) isinf(x) && return (x < 0 ? -one(T) : one(T))//zero(T) diff --git a/base/reduce.jl b/base/reduce.jl index 18ef01b4d63f4..fe8b7e3320da4 100644 --- a/base/reduce.jl +++ b/base/reduce.jl @@ -502,7 +502,7 @@ Count the number of elements in `itr` for which predicate `p` returns `true`. ```jldoctest julia> count(i->(4<=i<=6), [2,3,4,5,6]) - 3 +3 ``` """ function count(pred, itr) diff --git a/base/special/bessel.jl b/base/special/bessel.jl index 286770488759e..3151306de7c6a 100644 --- a/base/special/bessel.jl +++ b/base/special/bessel.jl @@ -40,6 +40,11 @@ let end end +""" + airy(k,x) + +The `k`th derivative of the Airy function ``\\operatorname{Ai}(x)``. +""" function airy(k::Integer, z::Complex128) id = Int32(k==1 || k==3) if k == 0 || k == 1 @@ -51,14 +56,43 @@ function airy(k::Integer, z::Complex128) end end +""" + airyprime(x) + +Airy function derivative ``\\operatorname{Ai}'(x)``. +""" airyprime(z) = airy(1,z) @vectorize_1arg Number airyprime + +""" + airyai(x) + +Airy function ``\\operatorname{Ai}(x)``. +""" airyai(z) = airy(0,z) @vectorize_1arg Number airyai + +""" + airyaiprime(x) + +Airy function derivative ``\\operatorname{Ai}'(x)``. +""" airyaiprime(z) = airy(1,z) @vectorize_1arg Number airyaiprime + +""" + airybi(x) + +Airy function ``\\operatorname{Bi}(x)``. +""" airybi(z) = airy(2,z) @vectorize_1arg Number airybi + +""" + airybiprime(x) + +Airy function derivative ``\\operatorname{Bi}'(x)``. +""" airybiprime(z) = airy(3,z) @vectorize_1arg Number airybiprime @@ -87,6 +121,14 @@ for afn in (:airy,:airyx) @vectorize_2arg Number $afn end end +""" + airyx(k,x) + +scaled `k`th derivative of the Airy function, return ``\\operatorname{Ai}(x) e^{\\frac{2}{3} x \\sqrt{x}}`` +for `k == 0 || k == 1`, and ``\\operatorname{Ai}(x) e^{- \\left| \\operatorname{Re} \\left( \\frac{2}{3} x \\sqrt{x} \\right) \\right|}`` +for `k == 2 || k == 3`. +""" +function airyx(k,x) end ## Bessel functions @@ -194,7 +236,8 @@ end besselh(nu, [k=1,] x) Bessel function of the third kind of order `nu` (the Hankel function). `k` is either 1 or 2, -selecting `hankelh1` or `hankelh2`, respectively. `k` defaults to 1 if it is omitted. +selecting [`hankelh1`](:func:`hankelh1`) or [`hankelh2`](:func:`hankelh2`), respectively. +`k` defaults to 1 if it is omitted. (See also [`besselhx`](:func:`besselhx`) for an exponentially scaled variant.) """ function besselh end @@ -299,7 +342,11 @@ function besselyx(nu::Float64, z::Complex128) end end +""" + besseli(nu, x) +Modified Bessel function of the first kind of order `nu`, ``I_\\nu(x)``. +""" function besseli(nu::Real, x::AbstractFloat) if x < 0 && !isinteger(nu) throw(DomainError()) @@ -307,6 +354,11 @@ function besseli(nu::Real, x::AbstractFloat) real(besseli(float(nu), complex(x))) end +""" + besselix(nu, x) + +Scaled modified Bessel function of the first kind of order `nu`, ``I_\\nu(x) e^{- | \\operatorname{Re}(x) |}``. +""" function besselix(nu::Real, x::AbstractFloat) if x < 0 && !isinteger(nu) throw(DomainError()) @@ -314,6 +366,11 @@ function besselix(nu::Real, x::AbstractFloat) real(besselix(float(nu), complex(x))) end +""" + besselj(nu, x) + +Bessel function of the first kind of order `nu`, ``J_\\nu(x)``. +""" function besselj(nu::Real, x::AbstractFloat) if isinteger(nu) if typemin(Cint) <= nu <= typemax(Cint) @@ -325,6 +382,11 @@ function besselj(nu::Real, x::AbstractFloat) real(besselj(float(nu), complex(x))) end +""" + besseljx(nu, x) + +Scaled Bessel function of the first kind of order `nu`, ``J_\\nu(x) e^{- | \\operatorname{Im}(x) |}``. +""" function besseljx(nu::Real, x::AbstractFloat) if x < 0 && !isinteger(nu) throw(DomainError()) @@ -332,6 +394,11 @@ function besseljx(nu::Real, x::AbstractFloat) real(besseljx(float(nu), complex(x))) end +""" + besselk(nu, x) + +Modified Bessel function of the second kind of order `nu`, ``K_\\nu(x)``. +""" function besselk(nu::Real, x::AbstractFloat) if x < 0 throw(DomainError()) @@ -341,6 +408,11 @@ function besselk(nu::Real, x::AbstractFloat) real(besselk(float(nu), complex(x))) end +""" + besselkx(nu, x) + +Scaled modified Bessel function of the second kind of order `nu`, ``K_\\nu(x) e^x``. +""" function besselkx(nu::Real, x::AbstractFloat) if x < 0 throw(DomainError()) @@ -350,6 +422,11 @@ function besselkx(nu::Real, x::AbstractFloat) real(besselkx(float(nu), complex(x))) end +""" + bessely(nu, x) + +Bessel function of the second kind of order `nu`, ``Y_\\nu(x)``. +""" function bessely(nu::Real, x::AbstractFloat) if x < 0 throw(DomainError()) @@ -359,6 +436,12 @@ function bessely(nu::Real, x::AbstractFloat) real(bessely(float(nu), complex(x))) end +""" + besselyx(nu, x) + +Scaled Bessel function of the second kind of order `nu`, +``Y_\\nu(x) e^{- | \\operatorname{Im}(x) |}``. +""" function besselyx(nu::Real, x::AbstractFloat) if x < 0 throw(DomainError()) @@ -398,15 +481,34 @@ for bfn in (:besselh, :besselhx) end end +""" + hankelh1(nu, x) +Bessel function of the third kind of order `nu`, ``H^{(1)}_\\nu(x)``. +""" hankelh1(nu, z) = besselh(nu, 1, z) @vectorize_2arg Number hankelh1 +""" + hankelh2(nu, x) + +Bessel function of the third kind of order `nu`, ``H^{(2)}_\\nu(x)``. +""" hankelh2(nu, z) = besselh(nu, 2, z) @vectorize_2arg Number hankelh2 +""" + hankelh1x(nu, x) + +Scaled Bessel function of the third kind of order `nu`, ``H^{(1)}_\\nu(x) e^{-x i}``. +""" hankelh1x(nu, z) = besselhx(nu, 1, z) @vectorize_2arg Number hankelh1x +""" + hankelh2x(nu, x) + +Scaled Bessel function of the third kind of order `nu`, ``H^{(2)}_\\nu(x) e^{x i}``. +""" hankelh2x(nu, z) = besselhx(nu, 2, z) @vectorize_2arg Number hankelh2x diff --git a/base/statistics.jl b/base/statistics.jl index d85569ec52526..7156c1d2a373e 100644 --- a/base/statistics.jl +++ b/base/statistics.jl @@ -37,6 +37,15 @@ momenttype{T}(::Type{T}) = typeof((zero(T) + zero(T)) / 2) momenttype(::Type{Float32}) = Float32 momenttype{T<:Union{Float64,Int32,Int64,UInt32,UInt64}}(::Type{T}) = Float64 +""" + mean(v[, region]) + +Compute the mean of whole array `v`, or optionally along the dimensions in `region`. + +!!! note + Julia does not ignore `NaN` values in the computation. For applications requiring the + handling of missing data, the `DataArrays.jl` package is recommended. +""" mean{T}(A::AbstractArray{T}, region) = mean!(reducedim_initarray(A, region, 0, momenttype(T)), A) @@ -147,6 +156,19 @@ function varm!{S}(R::AbstractArray{S}, A::AbstractArray, m::AbstractArray; corre return R end +""" + varm(v, m[, region]; corrected::Bool=true) + +Compute the sample variance of a collection `v` with known mean(s) `m`, +optionally over `region`. `m` may contain means for each dimension of +`v`. If `corrected` is `true`, then the sum is scaled with `n-1`, +whereas the sum is scaled with `n` if `corrected` is `false` where `n = length(x)`. + +!!! note + Julia does not ignore `NaN` values in the computation. For + applications requiring the handling of missing data, the + `DataArrays.jl` package is recommended. +""" varm{T}(A::AbstractArray{T}, m::AbstractArray, region; corrected::Bool=true) = varm!(reducedim_initarray(A, region, 0, real(momenttype(T))), A, m; corrected=corrected) @@ -198,12 +220,41 @@ stdm(A::AbstractArray, m::Number; corrected::Bool=true) = std(A::AbstractArray; corrected::Bool=true, mean=nothing) = sqrt(var(A; corrected=corrected, mean=mean)) +""" + std(v[, region]; corrected::Bool=true, mean=nothing) + +Compute the sample standard deviation of a vector or array `v`, optionally along dimensions +in `region`. The algorithm returns an estimator of the generative distribution's standard +deviation under the assumption that each entry of `v` is an IID drawn from that generative +distribution. This computation is equivalent to calculating `sqrt(sum((v - mean(v)).^2) / +(length(v) - 1))`. A pre-computed `mean` may be provided. If `corrected` is `true`, +then the sum is scaled with `n-1`, whereas the sum is scaled with `n` if `corrected` is +`false` where `n = length(x)`. + +!!! note + Julia does not ignore `NaN` values in the computation. For + applications requiring the handling of missing data, the + `DataArrays.jl` package is recommended. +""" std(A::AbstractArray, region; corrected::Bool=true, mean=nothing) = sqrt!(var(A, region; corrected=corrected, mean=mean)) std(iterable; corrected::Bool=true, mean=nothing) = sqrt(var(iterable, corrected=corrected, mean=mean)) +""" + stdm(v, m::Number; corrected::Bool=true) + +Compute the sample standard deviation of a vector `v` +with known mean `m`. If `corrected` is `true`, +then the sum is scaled with `n-1`, whereas the sum is +scaled with `n` if `corrected` is `false` where `n = length(x)`. + +!!! note + Julia does not ignore `NaN` values in the computation. For + applications requiring the handling of missing data, the + `DataArrays.jl` package is recommended. +""" stdm(iterable, m::Number; corrected::Bool=true) = std(iterable, corrected=corrected, mean=m) @@ -266,7 +317,7 @@ covm(x::AbstractVecOrMat, xmean, y::AbstractVecOrMat, ymean, vardim::Int=1, corr cov(x[, corrected=true]) Compute the variance of the vector `x`. If `corrected` is `true` (the default) then the sum -is scaled with `n-1` wheares the sum is scaled with `n` if `corrected` is `false` where `n = length(x)`. +is scaled with `n-1`, whereas the sum is scaled with `n` if `corrected` is `false` where `n = length(x)`. """ cov(x::AbstractVector, corrected::Bool) = covm(x, Base.mean(x), corrected) # This ugly hack is necessary to make the method below considered more specific than the deprecated method. When the old keyword version has been completely deprecated, these two methods can be merged @@ -276,7 +327,7 @@ cov{T<:AbstractVector}(x::T) = covm(x, Base.mean(x), true) cov(X[, vardim=1, corrected=true]) Compute the covariance matrix of the matrix `X` along the dimension `vardim`. If `corrected` -is `true` (the default) then the sum is scaled with `n-1` wheares the sum is scaled with `n` +is `true` (the default) then the sum is scaled with `n-1`, whereas the sum is scaled with `n` if `corrected` is `false` where `n = size(X, vardim)`. """ cov(X::AbstractMatrix, vardim::Int, corrected::Bool=true) = @@ -288,7 +339,7 @@ cov{T<:AbstractMatrix}(X::T) = cov(X, 1, true) cov(x, y[, corrected=true]) Compute the covariance between the vectors `x` and `y`. If `corrected` is `true` (the default) -then the sum is scaled with `n-1` wheares the sum is scaled with `n` if `corrected` is `false` +then the sum is scaled with `n-1`, whereas the sum is scaled with `n` if `corrected` is `false` where `n = length(x) = length(y)`. """ cov(x::AbstractVector, y::AbstractVector, corrected::Bool) = @@ -301,7 +352,7 @@ cov{T<:AbstractVector,S<:AbstractVector}(x::T, y::S) = cov(X, Y[, vardim=1, corrected=true]) Compute the covariance between the vectors or matrices `X` and `Y` along the dimension -`vardim`. If `corrected` is `true` (the default) then the sum is scaled with `n-1` wheares +`vardim`. If `corrected` is `true` (the default) then the sum is scaled with `n-1`, whereas the sum is scaled with `n` if `corrected` is `false` where `n = size(X, vardim) = size(Y, vardim)`. """ cov(X::AbstractVecOrMat, Y::AbstractVecOrMat, vardim::Int, corrected::Bool=true) = @@ -457,21 +508,41 @@ middle(x::Real) = (x + zero(x)) / 1 """ middle(x, y) -Compute the middle of two reals `x` and `y`, which is equivalent in both value and type to computing their mean (`(x + y) / 2`). +Compute the middle of two reals `x` and `y`, which is +equivalent in both value and type to computing their mean (`(x + y) / 2`). """ middle(x::Real, y::Real) = x/2 + y/2 """ middle(range) -Compute the middle of a range, which consists in computing the mean of its extrema. Since a range is sorted, the mean is performed with the first and last element. +Compute the middle of a range, which consists of computing the mean of its extrema. +Since a range is sorted, the mean is performed with the first and last element. + +```jldoctest +julia> middle(1:10) +5.5 +``` """ middle(a::Range) = middle(a[1], a[end]) """ - middle(array) - -Compute the middle of an array, which consists in finding its extrema and then computing their mean. + middle(a) + +Compute the middle of an array `a`, which consists of finding its +extrema and then computing their mean. + +```jldoctest +julia> a = [1,2,3.6,10.9] +4-element Array{Float64,1}: + 1.0 + 2.0 + 3.6 + 10.9 + +julia> middle(a) +5.95 +``` """ middle(a::AbstractArray) = ((v1, v2) = extrema(a); middle(v1, v2)) @@ -493,6 +564,18 @@ end median!{T}(v::AbstractArray{T}) = median!(vec(v)) median{T}(v::AbstractArray{T}) = median!(copy!(Array{T,1}(length(v)), v)) +""" + median(v[, region]) + +Compute the median of an entire array `v`, or, optionally, +along the dimensions in `region`. For an even number of +elements no exact median element exists, so the result is +equivalent to calculating mean of two median elements. + +!!! note + Julia does not ignore `NaN` values in the computation. For applications requiring the + handling of missing data, the `DataArrays.jl` package is recommended. +""" median{T}(v::AbstractArray{T}, region) = mapslices(median!, v, region) # for now, use the R/S definition of quantile; may want variants later @@ -512,12 +595,19 @@ Quantiles are computed via linear interpolation between the points `((k-1)/(n-1) for `k = 1:n` where `n = length(v)`. This corresponds to Definition 7 of Hyndman and Fan (1996), and is the same as the R default. +!!! note + Julia does not ignore `NaN` values in the computation. For applications requiring the + handling of missing data, the `DataArrays.jl` package is recommended. `quantile!` will + throw an `ArgumentError` in the presence of `NaN` values in the data array. + * Hyndman, R.J and Fan, Y. (1996) "Sample Quantiles in Statistical Packages", *The American Statistician*, Vol. 50, No. 4, pp. 361-365 """ function quantile!(q::AbstractArray, v::AbstractVector, p::AbstractArray; sorted::Bool=false) - size(p) == size(q) || throw(DimensionMismatch()) + if size(p) != size(q) + throw(DimensionMismatch("size of p, $(size(p)), must equal size of q, $(size(q))")) + end isempty(v) && throw(ArgumentError("empty data vector")) @@ -593,6 +683,11 @@ Quantiles are computed via linear interpolation between the points `((k-1)/(n-1) for `k = 1:n` where `n = length(v)`. This corresponds to Definition 7 of Hyndman and Fan (1996), and is the same as the R default. +!!! note + Julia does not ignore `NaN` values in the computation. For applications requiring the + handling of missing data, the `DataArrays.jl` package is recommended. `quantile` will + throw an `ArgumentError` in the presence of `NaN` values in the data array. + * Hyndman, R.J and Fan, Y. (1996) "Sample Quantiles in Statistical Packages", *The American Statistician*, Vol. 50, No. 4, pp. 361-365 """ diff --git a/doc/manual/arrays.rst b/doc/manual/arrays.rst index e04ae53de2ae0..5644e2d047bf6 100644 --- a/doc/manual/arrays.rst +++ b/doc/manual/arrays.rst @@ -517,7 +517,7 @@ the name of the function to vectorize. Here is a simple example: julia> methods(square) # 2 methods for generic function "square": - square{T<:Number}(x::AbstractArray{T,N<:Any}) at operators.jl:540 + square{T<:Number}(x::AbstractArray{T,N<:Any}) at operators.jl:550 square(x) at none:1 julia> square([1 2 4; 5 6 7]) diff --git a/doc/manual/complex-and-rational-numbers.rst b/doc/manual/complex-and-rational-numbers.rst index aa9cd00783e47..ae30e44e838eb 100644 --- a/doc/manual/complex-and-rational-numbers.rst +++ b/doc/manual/complex-and-rational-numbers.rst @@ -159,7 +159,7 @@ versus ``-1 + 0im`` even though ``-1 == -1 + 0im``: julia> sqrt(-1) ERROR: DomainError: sqrt will only return a complex result if called with a complex argument. Try sqrt(complex(x)). - in sqrt(::Int64) at ./math.jl:169 + in sqrt(::Int64) at ./math.jl:185 ... julia> sqrt(-1 + 0im) diff --git a/doc/manual/control-flow.rst b/doc/manual/control-flow.rst index 0a1180581594b..4168cb0118ce0 100644 --- a/doc/manual/control-flow.rst +++ b/doc/manual/control-flow.rst @@ -666,7 +666,7 @@ negative real value: julia> sqrt(-1) ERROR: DomainError: sqrt will only return a complex result if called with a complex argument. Try sqrt(complex(x)). - in sqrt(::Int64) at ./math.jl:169 + in sqrt(::Int64) at ./math.jl:185 ... You may define your own exceptions in the following way: diff --git a/doc/manual/strings.rst b/doc/manual/strings.rst index a1838fd57e903..e8e3c0813e843 100644 --- a/doc/manual/strings.rst +++ b/doc/manual/strings.rst @@ -553,7 +553,7 @@ contained in a string: julia> contains("Xylophon", 'o') ERROR: MethodError: no method matching contains(::String, ::Char) Closest candidates are: - contains(!Matched::Function, ::Any, !Matched::Any) at reduce.jl:402 + contains(!Matched::Function, ::Any, !Matched::Any) at reduce.jl:489 contains(::AbstractString, !Matched::AbstractString) at strings/search.jl:310 ... diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index f72f67f72b10a..f45ff006269a5 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -437,12 +437,75 @@ Indexing, Assignment, and Concatenation Concatenate along dimension 1. + .. doctest:: + + julia> a = [1 2 3 4 5] + 1×5 Array{Int64,2}: + 1 2 3 4 5 + + julia> b = [6 7 8 9 10; 11 12 13 14 15] + 2×5 Array{Int64,2}: + 6 7 8 9 10 + 11 12 13 14 15 + + julia> vcat(a,b) + 3×5 Array{Int64,2}: + 1 2 3 4 5 + 6 7 8 9 10 + 11 12 13 14 15 + + julia> c = ([1 2 3], [4 5 6]) + ( + [1 2 3], + + [4 5 6]) + + julia> vcat(c...) + 2×3 Array{Int64,2}: + 1 2 3 + 4 5 6 + .. function:: hcat(A...) .. Docstring generated from Julia source Concatenate along dimension 2. + .. doctest:: + + julia> a = [1; 2; 3; 4; 5] + 5-element Array{Int64,1}: + 1 + 2 + 3 + 4 + 5 + + julia> b = [6 7; 8 9; 10 11; 12 13; 14 15] + 5×2 Array{Int64,2}: + 6 7 + 8 9 + 10 11 + 12 13 + 14 15 + + julia> hcat(a,b) + 5×3 Array{Int64,2}: + 1 6 7 + 2 8 9 + 3 10 11 + 4 12 13 + 5 14 15 + + julia> c = ([1; 2; 3], [4; 5; 6]) + ([1,2,3],[4,5,6]) + + julia> hcat(c...) + 3×2 Array{Int64,2}: + 1 4 + 2 5 + 3 6 + .. function:: hvcat(rows::Tuple{Vararg{Int}}, values...) .. Docstring generated from Julia source @@ -484,12 +547,47 @@ Indexing, Assignment, and Concatenation Reverse ``A`` in dimension ``d``\ . -.. function:: circshift(A,shifts) + .. doctest:: + + julia> b = [1 2; 3 4] + 2×2 Array{Int64,2}: + 1 2 + 3 4 + + julia> flipdim(b,2) + 2×2 Array{Int64,2}: + 2 1 + 4 3 + +.. function:: circshift(A, shifts) .. Docstring generated from Julia source Circularly shift the data in an array. The second argument is a vector giving the amount to shift in each dimension. + .. doctest:: + + julia> b = reshape(collect(1:16), (4,4)) + 4×4 Array{Int64,2}: + 1 5 9 13 + 2 6 10 14 + 3 7 11 15 + 4 8 12 16 + + julia> circshift(b, [0,2]) + 4×4 Array{Int64,2}: + 9 13 1 5 + 10 14 2 6 + 11 15 3 7 + 12 16 4 8 + + julia> circshift(b, [-1,0]) + 4×4 Array{Int64,2}: + 2 6 10 14 + 3 7 11 15 + 4 8 12 16 + 1 5 9 13 + .. function:: find(A) .. Docstring generated from Julia source @@ -608,13 +706,43 @@ Indexing, Assignment, and Concatenation .. Docstring generated from Julia source - Remove the dimensions specified by ``dims`` from array ``A``\ . Elements of ``dims`` must be unique and within the range ``1:ndims(A)``\ . + Remove the dimensions specified by ``dims`` from array ``A``\ . Elements of ``dims`` must be unique and within the range ``1:ndims(A)``\ . ``size(A,i)`` must equal 1 for all ``i`` in ``dims``\ . -.. function:: vec(Array) -> Vector + .. doctest:: + + julia> a = reshape(collect(1:4),(2,2,1,1)) + 2×2×1×1 Array{Int64,4}: + [:, :, 1, 1] = + 1 3 + 2 4 + + julia> squeeze(a,3) + 2×2×1 Array{Int64,3}: + [:, :, 1] = + 1 3 + 2 4 + +.. function:: vec(a::AbstractArray) -> Vector .. Docstring generated from Julia source - Vectorize an array using column-major convention. + Reshape array ``a`` as a one-dimensional column vector. + + .. doctest:: + + julia> a = [1 2 3; 4 5 6] + 2×3 Array{Int64,2}: + 1 2 3 + 4 5 6 + + julia> vec(a) + 6-element Array{Int64,1}: + 1 + 4 + 2 + 5 + 3 + 6 .. function:: promote_shape(s1, s2) diff --git a/doc/stdlib/collections.rst b/doc/stdlib/collections.rst index 4cc7c6eb4be22..e37157208b22a 100644 --- a/doc/stdlib/collections.rst +++ b/doc/stdlib/collections.rst @@ -124,13 +124,13 @@ type. 5 7 9 - 11 + 11 julia> collect(take(a,3)) 3-element Array{Int64,1}: - 1 - 3 - 5 + 1 + 3 + 5 .. function:: drop(iter, n) @@ -150,12 +150,12 @@ type. 5 7 9 - 11 + 11 julia> collect(drop(a,4)) 2-element Array{Int64,1}: 9 - 11 + 11 .. function:: cycle(iter) @@ -295,7 +295,7 @@ Iterable Collections julia> a = ['a', 'b', 'c', 'b', 'd', 'a']; - julia> b = ['a','b','c'] + julia> b = ['a','b','c']; julia> indexin(a,b) 6-element Array{Int64,1}: @@ -523,7 +523,7 @@ Iterable Collections .. doctest:: - julia> findmax([8,0.1,-9,pi]) + julia> findmin([8,0.1,-9,pi]) (-9.0,3) .. function:: findmin(A, dims) -> (minval, index) @@ -713,7 +713,7 @@ Iterable Collections .. doctest:: julia> count(i->(4<=i<=6), [2,3,4,5,6]) - 3 + 3 .. function:: any(p, itr) -> Bool @@ -745,8 +745,7 @@ Iterable Collections .. doctest:: - julia> a - 1:3:7 + julia> a = 1:3:7; julia> foreach(x->println(x^2),a) 1 diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index 47620a1b53e14..7c9a7c653e808 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -201,11 +201,22 @@ Mathematical Operators Divide two integers or rational numbers, giving a ``Rational`` result. -.. function:: rationalize([Type=Int,] x; tol=eps(x)) +.. function:: rationalize([T<:Integer=Int,] x; tol::Real=eps(x)) .. Docstring generated from Julia source - Approximate floating point number ``x`` as a Rational number with components of the given integer type. The result will differ from ``x`` by no more than ``tol``\ . + Approximate floating point number ``x`` as a ``Rational`` number with components of the given integer type. The result will differ from ``x`` by no more than ``tol``\ . If ``T`` is not provided, it defaults to ``Int``\ . + + .. doctest:: + + julia> rationalize(5.6) + 28//5 + + julia> a = rationalize(BigInt, 10.3) + 103//10 + + julia> typeof(num(a)) + BigInt .. function:: num(x) @@ -1021,13 +1032,21 @@ Mathematical Functions .. Docstring generated from Julia source - Return ``x`` if ``lo <= x <= hi``\ . If ``x < lo``\ , return ``lo``\ . If ``x > hi``\ , return ``hi``\ . Arguments are promoted to a common type. Operates elementwise over ``x`` if it is an array. + Return ``x`` if ``lo <= x <= hi``\ . If ``x < lo``\ , return ``lo``\ . If ``x > hi``\ , return ``hi``\ . Arguments are promoted to a common type. Operates elementwise over ``x`` if ``x`` is an array. + + .. doctest:: + + julia> clamp([pi, 1.0, big(10.)], 2., 9.) + 3-element Array{BigFloat,1}: + 3.141592653589793238462643383279502884197169399375105820974944592307816406286198 + 2.000000000000000000000000000000000000000000000000000000000000000000000000000000 + 9.000000000000000000000000000000000000000000000000000000000000000000000000000000 .. function:: clamp!(array::AbstractArray, lo, hi) .. Docstring generated from Julia source - Restrict values in ``array`` to the specified range, in-place. + Restrict values in ``array`` to the specified range, in-place. See also :func:`clamp`\ . .. function:: abs(x) @@ -1495,7 +1514,7 @@ Mathematical Functions .. Docstring generated from Julia source - Bessel function of the third kind of order ``nu`` (the Hankel function). ``k`` is either 1 or 2, selecting ``hankelh1`` or ``hankelh2``\ , respectively. ``k`` defaults to 1 if it is omitted. (See also :func:`besselhx` for an exponentially scaled variant.) + Bessel function of the third kind of order ``nu`` (the Hankel function). ``k`` is either 1 or 2, selecting :func:`hankelh1` or :func:`hankelh2`\ , respectively. ``k`` defaults to 1 if it is omitted. (See also :func:`besselhx` for an exponentially scaled variant.) .. function:: besselhx(nu, [k=1,] z) @@ -1584,7 +1603,11 @@ Statistics .. Docstring generated from Julia source - Compute the mean of whole array ``v``\ , or optionally along the dimensions in ``region``\ . Note: Julia does not ignore ``NaN`` values in the computation. For applications requiring the handling of missing data, the ``DataArray`` package is recommended. + Compute the mean of whole array ``v``\ , or optionally along the dimensions in ``region``\ . + + .. note:: + Julia does not ignore ``NaN`` values in the computation. For applications requiring the handling of missing data, the ``DataArrays.jl`` package is recommended. + .. function:: mean(f::Function, v) @@ -1598,17 +1621,25 @@ Statistics Compute the mean of ``v`` over the singleton dimensions of ``r``\ , and write results to ``r``\ . -.. function:: std(v[, region]) +.. function:: std(v[, region]; corrected::Bool=true, mean=nothing) .. Docstring generated from Julia source - Compute the sample standard deviation of a vector or array ``v``\ , optionally along dimensions in ``region``\ . The algorithm returns an estimator of the generative distribution's standard deviation under the assumption that each entry of ``v`` is an IID drawn from that generative distribution. This computation is equivalent to calculating ``sqrt(sum((v - mean(v)).^2) / (length(v) - 1))``\ . Note: Julia does not ignore ``NaN`` values in the computation. For applications requiring the handling of missing data, the ``DataArray`` package is recommended. + Compute the sample standard deviation of a vector or array ``v``\ , optionally along dimensions in ``region``\ . The algorithm returns an estimator of the generative distribution's standard deviation under the assumption that each entry of ``v`` is an IID drawn from that generative distribution. This computation is equivalent to calculating ``sqrt(sum((v - mean(v)).^2) / (length(v) - 1))``\ . A pre-computed ``mean`` may be provided. If ``corrected`` is ``true``\ , then the sum is scaled with ``n-1``\ , whereas the sum is scaled with ``n`` if ``corrected`` is ``false`` where ``n = length(x)``\ . -.. function:: stdm(v, m) + .. note:: + Julia does not ignore ``NaN`` values in the computation. For applications requiring the handling of missing data, the ``DataArrays.jl`` package is recommended. + + +.. function:: stdm(v, m::Number; corrected::Bool=true) .. Docstring generated from Julia source - Compute the sample standard deviation of a vector ``v`` with known mean ``m``\ . Note: Julia does not ignore ``NaN`` values in the computation. + Compute the sample standard deviation of a vector ``v`` with known mean ``m``\ . If ``corrected`` is ``true``\ , then the sum is scaled with ``n-1``\ , whereas the sum is scaled with ``n`` if ``corrected`` is ``false`` where ``n = length(x)``\ . + + .. note:: + Julia does not ignore ``NaN`` values in the computation. For applications requiring the handling of missing data, the ``DataArrays.jl`` package is recommended. + .. function:: var(v[, region]) @@ -1616,11 +1647,15 @@ Statistics Compute the sample variance of a vector or array ``v``\ , optionally along dimensions in ``region``\ . The algorithm will return an estimator of the generative distribution's variance under the assumption that each entry of ``v`` is an IID drawn from that generative distribution. This computation is equivalent to calculating ``sumabs2(v - mean(v)) / (length(v) - 1)``\ . Note: Julia does not ignore ``NaN`` values in the computation. For applications requiring the handling of missing data, the ``DataArray`` package is recommended. -.. function:: varm(v, m) +.. function:: varm(v, m[, region]; corrected::Bool=true) .. Docstring generated from Julia source - Compute the sample variance of a vector ``v`` with known mean ``m``\ . Note: Julia does not ignore ``NaN`` values in the computation. + Compute the sample variance of a collection ``v`` with known mean(s) ``m``\ , optionally over ``region``\ . ``m`` may contain means for each dimension of ``v``\ . If ``corrected`` is ``true``\ , then the sum is scaled with ``n-1``\ , whereas the sum is scaled with ``n`` if ``corrected`` is ``false`` where ``n = length(x)``\ . + + .. note:: + Julia does not ignore ``NaN`` values in the computation. For applications requiring the handling of missing data, the ``DataArrays.jl`` package is recommended. + .. function:: middle(x) @@ -1638,19 +1673,40 @@ Statistics .. Docstring generated from Julia source - Compute the middle of a range, which consists in computing the mean of its extrema. Since a range is sorted, the mean is performed with the first and last element. + Compute the middle of a range, which consists of computing the mean of its extrema. Since a range is sorted, the mean is performed with the first and last element. + + .. doctest:: + + julia> middle(1:10) + 5.5 -.. function:: middle(array) +.. function:: middle(a) .. Docstring generated from Julia source - Compute the middle of an array, which consists in finding its extrema and then computing their mean. + Compute the middle of an array ``a``\ , which consists of finding its extrema and then computing their mean. + + .. doctest:: + + julia> a = [1,2,3.6,10.9] + 4-element Array{Float64,1}: + 1.0 + 2.0 + 3.6 + 10.9 + + julia> middle(a) + 5.95 .. function:: median(v[, region]) .. Docstring generated from Julia source - Compute the median of whole array ``v``\ , or optionally along the dimensions in ``region``\ . For even number of elements no exact median element exists, so the result is equivalent to calculating mean of two median elements. ``NaN`` is returned if the data contains any ``NaN`` values. For applications requiring the handling of missing data, the ``DataArrays`` package is recommended. + Compute the median of an entire array ``v``\ , or, optionally, along the dimensions in ``region``\ . For an even number of elements no exact median element exists, so the result is equivalent to calculating mean of two median elements. + + .. note:: + Julia does not ignore ``NaN`` values in the computation. For applications requiring the handling of missing data, the ``DataArrays.jl`` package is recommended. + .. function:: median!(v) @@ -1674,6 +1730,10 @@ Statistics Quantiles are computed via linear interpolation between the points ``((k-1)/(n-1), v[k])``\ , for ``k = 1:n`` where ``n = length(v)``\ . This corresponds to Definition 7 of Hyndman and Fan (1996), and is the same as the R default. + .. note:: + Julia does not ignore ``NaN`` values in the computation. For applications requiring the handling of missing data, the ``DataArrays.jl`` package is recommended. ``quantile`` will throw an ``ArgumentError`` in the presence of ``NaN`` values in the data array. + + * Hyndman, R.J and Fan, Y. (1996) "Sample Quantiles in Statistical Packages", *The American Statistician*, Vol. 50, No. 4, pp. 361-365 .. function:: quantile!([q, ] v, p; sorted=false) @@ -1686,31 +1746,35 @@ Statistics Quantiles are computed via linear interpolation between the points ``((k-1)/(n-1), v[k])``\ , for ``k = 1:n`` where ``n = length(v)``\ . This corresponds to Definition 7 of Hyndman and Fan (1996), and is the same as the R default. + .. note:: + Julia does not ignore ``NaN`` values in the computation. For applications requiring the handling of missing data, the ``DataArrays.jl`` package is recommended. ``quantile!`` will throw an ``ArgumentError`` in the presence of ``NaN`` values in the data array. + + * Hyndman, R.J and Fan, Y. (1996) "Sample Quantiles in Statistical Packages", *The American Statistician*, Vol. 50, No. 4, pp. 361-365 .. function:: cov(x[, corrected=true]) .. Docstring generated from Julia source - Compute the variance of the vector ``x``\ . If ``corrected`` is ``true`` (the default) then the sum is scaled with ``n-1`` wheares the sum is scaled with ``n`` if ``corrected`` is ``false`` where ``n = length(x)``\ . + Compute the variance of the vector ``x``\ . If ``corrected`` is ``true`` (the default) then the sum is scaled with ``n-1``\ , whereas the sum is scaled with ``n`` if ``corrected`` is ``false`` where ``n = length(x)``\ . .. function:: cov(X[, vardim=1, corrected=true]) .. Docstring generated from Julia source - Compute the covariance matrix of the matrix ``X`` along the dimension ``vardim``\ . If ``corrected`` is ``true`` (the default) then the sum is scaled with ``n-1`` wheares the sum is scaled with ``n`` if ``corrected`` is ``false`` where ``n = size(X, vardim)``\ . + Compute the covariance matrix of the matrix ``X`` along the dimension ``vardim``\ . If ``corrected`` is ``true`` (the default) then the sum is scaled with ``n-1``\ , whereas the sum is scaled with ``n`` if ``corrected`` is ``false`` where ``n = size(X, vardim)``\ . .. function:: cov(x, y[, corrected=true]) .. Docstring generated from Julia source - Compute the covariance between the vectors ``x`` and ``y``\ . If ``corrected`` is ``true`` (the default) then the sum is scaled with ``n-1`` wheares the sum is scaled with ``n`` if ``corrected`` is ``false`` where ``n = length(x) = length(y)``\ . + Compute the covariance between the vectors ``x`` and ``y``\ . If ``corrected`` is ``true`` (the default) then the sum is scaled with ``n-1``\ , whereas the sum is scaled with ``n`` if ``corrected`` is ``false`` where ``n = length(x) = length(y)``\ . .. function:: cov(X, Y[, vardim=1, corrected=true]) .. Docstring generated from Julia source - Compute the covariance between the vectors or matrices ``X`` and ``Y`` along the dimension ``vardim``\ . If ``corrected`` is ``true`` (the default) then the sum is scaled with ``n-1`` wheares the sum is scaled with ``n`` if ``corrected`` is ``false`` where ``n = size(X, vardim) = size(Y, vardim)``\ . + Compute the covariance between the vectors or matrices ``X`` and ``Y`` along the dimension ``vardim``\ . If ``corrected`` is ``true`` (the default) then the sum is scaled with ``n-1``\ , whereas the sum is scaled with ``n`` if ``corrected`` is ``false`` where ``n = size(X, vardim) = size(Y, vardim)``\ . .. function:: cor(x) @@ -2047,11 +2111,19 @@ some built-in integration support in Julia. Returns a pair ``(I,E)`` of the estimated integral ``I`` and an estimated upper bound on the absolute error ``E``\ . If ``maxevals`` is not exceeded then ``E <= max(abstol, reltol*norm(I))`` will hold. (Note that it is useful to specify a positive ``abstol`` in cases where ``norm(I)`` may be zero.) - The endpoints ``a`` etcetera can also be complex (in which case the integral is performed over straight-line segments in the complex plane). If the endpoints are ``BigFloat``\ , then the integration will be performed in ``BigFloat`` precision as well (note: it is advisable to increase the integration ``order`` in rough proportion to the precision, for smooth integrands). More generally, the precision is set by the precision of the integration endpoints (promoted to floating-point types). + The endpoints ``a`` et cetera can also be complex (in which case the integral is performed over straight-line segments in the complex plane). If the endpoints are ``BigFloat``\ , then the integration will be performed in ``BigFloat`` precision as well. + + .. note:: + It is advisable to increase the integration ``order`` in rough proportion to the precision, for smooth integrands. + + + More generally, the precision is set by the precision of the integration endpoints (promoted to floating-point types). The integrand ``f(x)`` can return any numeric scalar, vector, or matrix type, or in fact any type supporting ``+``\ , ``-``\ , multiplication by real values, and a ``norm`` (i.e., any normed vector space). Alternatively, a different norm can be specified by passing a ``norm``\ -like function as the ``norm`` keyword argument (which defaults to ``vecnorm``\ ). - [Only one-dimensional integrals are provided by this function. For multi-dimensional integration (cubature), there are many different algorithms (often much better than simple nested 1d integrals) and the optimal choice tends to be very problem-dependent. See the Julia external-package listing for available algorithms for multidimensional integration or other specialized tasks (such as integrals of highly oscillatory or singular functions).] + .. note:: + Only one-dimensional integrals are provided by this function. For multi-dimensional integration (cubature), there are many different algorithms (often much better than simple nested 1d integrals) and the optimal choice tends to be very problem-dependent. See the Julia external-package listing for available algorithms for multidimensional integration or other specialized tasks (such as integrals of highly oscillatory or singular functions). + The algorithm is an adaptive Gauss-Kronrod integration technique: the integral in each interval is estimated using a Kronrod rule (``2*order+1`` points) and the error is estimated using an embedded Gauss rule (``order`` points). The interval with the largest error is then subdivided into two intervals and the process is repeated until the desired error tolerance is achieved. diff --git a/doc/stdlib/pkg.rst b/doc/stdlib/pkg.rst index 33d35f6ef11c3..f0d455b9e0829 100644 --- a/doc/stdlib/pkg.rst +++ b/doc/stdlib/pkg.rst @@ -13,7 +13,7 @@ Functions for package development (e.g. ``tag``, ``publish``, etc.) have been mo .. Docstring generated from Julia source - Returns the absolute path of the package directory. This defaults to ``joinpath(homedir(),".julia","v$(VERSION.major).$(VERSION.minor)")`` on all platforms (i.e. ``~/.julia/v0.5`` in UNIX shell syntax). If the ``JULIA_PKGDIR`` environment variable is set, then that path is used in the returned value as ``joinpath(ENV["JULIA_PKGDIR"],"v$(VERSION.major).$(VERSION.minor)")``\ . If ``JULIA_PKGDIR`` is a relative path, it is interpreted relative to whatever the current working directory is. + Returns the absolute path of the package directory. This defaults to ``joinpath(homedir(),".julia","v$(VERSION.major).$(VERSION.minor)")`` on all platforms (i.e. ``~/.julia/v0.6`` in UNIX shell syntax). If the ``JULIA_PKGDIR`` environment variable is set, then that path is used in the returned value as ``joinpath(ENV["JULIA_PKGDIR"],"v$(VERSION.major).$(VERSION.minor)")``\ . If ``JULIA_PKGDIR`` is a relative path, it is interpreted relative to whatever the current working directory is. .. function:: dir(names...) -> AbstractString From 730c0a3f2f69be1f937b4ba5a8cdafc13016909f Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Thu, 4 Aug 2016 07:28:12 -0500 Subject: [PATCH 09/80] indices extensions for iterators & collect (cherry picked from commit a1e5daff1e2f47a3f6e56b01c800cc4cd5b40637) ref #17816 --- base/array.jl | 4 ++-- base/iterator.jl | 19 ++++++++++++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/base/array.jl b/base/array.jl index f892f2f6689d7..f20d5ccd1a5cb 100644 --- a/base/array.jl +++ b/base/array.jl @@ -211,7 +211,7 @@ The result has the same shape and number of dimensions as `collection`. collect{T}(::Type{T}, itr) = _collect(T, itr, iteratorsize(itr)) _collect{T}(::Type{T}, itr, isz::HasLength) = copy!(Array{T,1}(Int(length(itr)::Integer)), itr) -_collect{T}(::Type{T}, itr, isz::HasShape) = copy!(Array{T}(convert(Dims,size(itr))), itr) +_collect{T}(::Type{T}, itr, isz::HasShape) = copy!(similar(Array{T}, indices(itr)), itr) function _collect{T}(::Type{T}, itr, isz::SizeUnknown) a = Array{T,1}(0) for x in itr @@ -259,7 +259,7 @@ end _default_eltype{I,T}(::Type{Generator{I,Type{T}}}) = T _array_for(T, itr, ::HasLength) = Array{T,1}(Int(length(itr)::Integer)) -_array_for(T, itr, ::HasShape) = Array{T}(convert(Dims,size(itr))) +_array_for(T, itr, ::HasShape) = similar(Array{T}, indices(itr)) function collect(itr::Generator) isz = iteratorsize(itr.iter) diff --git a/base/iterator.jl b/base/iterator.jl index 1e11264915a91..3d74fd5da57db 100644 --- a/base/iterator.jl +++ b/base/iterator.jl @@ -69,6 +69,7 @@ end zip(a) = Zip1(a) length(z::Zip1) = length(z.a) size(z::Zip1) = size(z.a) +indices(z::Zip1) = indices(z.a) eltype{I}(::Type{Zip1{I}}) = Tuple{eltype(I)} @inline start(z::Zip1) = start(z.a) @inline function next(z::Zip1, st) @@ -87,6 +88,7 @@ end zip(a, b) = Zip2(a, b) length(z::Zip2) = _min_length(z.a, z.b, iteratorsize(z.a), iteratorsize(z.b)) size(z::Zip2) = promote_shape(size(z.a), size(z.b)) +indices(z::Zip2) = promote_shape(indices(z.a), indices(z.b)) eltype{I1,I2}(::Type{Zip2{I1,I2}}) = Tuple{eltype(I1), eltype(I2)} @inline start(z::Zip2) = (start(z.a), start(z.b)) @inline function next(z::Zip2, st) @@ -137,6 +139,7 @@ julia> first(c) zip(a, b, c...) = Zip(a, zip(b, c...)) length(z::Zip) = _min_length(z.a, z.z, iteratorsize(z.a), iteratorsize(z.z)) size(z::Zip) = promote_shape(size(z.a), size(z.z)) +indices(z::Zip) = promote_shape(indices(z.a), indices(z.z)) tuple_type_cons{S}(::Type{S}, ::Type{Union{}}) = Union{} function tuple_type_cons{S,T<:Tuple}(::Type{S}, ::Type{T}) @_pure_meta @@ -430,8 +433,10 @@ iteratoreltype{O}(::Type{Repeated{O}}) = HasEltype() abstract AbstractProdIterator length(p::AbstractProdIterator) = prod(size(p)) +_length(p::AbstractProdIterator) = prod(map(unsafe_length, indices(p))) size(p::AbstractProdIterator) = _prod_size(p.a, p.b, iteratorsize(p.a), iteratorsize(p.b)) -ndims(p::AbstractProdIterator) = length(size(p)) +indices(p::AbstractProdIterator) = _prod_indices(p.a, p.b, iteratorsize(p.a), iteratorsize(p.b)) +ndims(p::AbstractProdIterator) = length(indices(p)) # generic methods to handle size of Prod* types _prod_size(a, ::HasShape) = size(a) @@ -445,6 +450,17 @@ _prod_size(a, b, ::HasShape, ::HasShape) = (size(a)..., size(b)...) _prod_size(a, b, A, B) = throw(ArgumentError("Cannot construct size for objects of types $(typeof(a)) and $(typeof(b))")) +_prod_indices(a, ::HasShape) = indices(a) +_prod_indices(a, ::HasLength) = (OneTo(length(a)), ) +_prod_indices(a, A) = + throw(ArgumentError("Cannot compute indices for object of type $(typeof(a))")) +_prod_indices(a, b, ::HasLength, ::HasLength) = (OneTo(length(a)), OneTo(length(b))) +_prod_indices(a, b, ::HasLength, ::HasShape) = (OneTo(length(a)), indices(b)...) +_prod_indices(a, b, ::HasShape, ::HasLength) = (indices(a)..., OneTo(length(b))) +_prod_indices(a, b, ::HasShape, ::HasShape) = (indices(a)..., indices(b)...) +_prod_indices(a, b, A, B) = + throw(ArgumentError("Cannot construct indices for objects of types $(typeof(a)) and $(typeof(b))")) + # one iterator immutable Prod1{I} <: AbstractProdIterator a::I @@ -453,6 +469,7 @@ product(a) = Prod1(a) eltype{I}(::Type{Prod1{I}}) = Tuple{eltype(I)} size(p::Prod1) = _prod_size(p.a, iteratorsize(p.a)) +indices(p::Prod1) = _prod_indices(p.a, iteratorsize(p.a)) @inline start(p::Prod1) = start(p.a) @inline function next(p::Prod1, st) From 1debda49e660ef19c98e69ae66ae3b33e07bf7fa Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Thu, 4 Aug 2016 07:30:18 -0500 Subject: [PATCH 10/80] Get (c)transpose for vectors working with non-1 indices It was already working for matrices (cherry picked from commit 9114ae6030e14847e8557eaf11ca6d985743ccee) ref #17816 --- base/abstractarray.jl | 7 +++++++ base/arraymath.jl | 4 ++-- test/offsetarray.jl | 17 ++++++++++++++++- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 7a6ead53c946a..dd58280e1de5f 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -625,6 +625,13 @@ convert{T,S,N}(::Type{AbstractArray{T }}, A::AbstractArray{S,N}) = convert(Abst convert{T,N}(::Type{Array}, A::AbstractArray{T,N}) = convert(Array{T,N}, A) +""" + of_indices(x, y) + +Represents the array `y` as an array having the same indices type as `x`. +""" +of_indices(x, y) = similar(dims->y, oftype(indices(x), indices(y))) + full(x::AbstractArray) = x map(::Type{Integer}, a::Array) = map!(Integer, similar(a,typeof(Integer(one(eltype(a))))), a) diff --git a/base/arraymath.jl b/base/arraymath.jl index 6d3a2d9770413..60b78f0a2fed3 100644 --- a/base/arraymath.jl +++ b/base/arraymath.jl @@ -411,8 +411,8 @@ function ctranspose(A::AbstractMatrix) end ctranspose{T<:Real}(A::AbstractVecOrMat{T}) = transpose(A) -transpose(x::AbstractVector) = [ transpose(v) for i=1:1, v in x ] -ctranspose{T}(x::AbstractVector{T}) = T[ ctranspose(v) for i=1:1, v in x ] +transpose(x::AbstractVector) = [ transpose(v) for i=of_indices(x, OneTo(1)), v in x ] +ctranspose{T}(x::AbstractVector{T}) = T[ ctranspose(v) for i=of_indices(x, OneTo(1)), v in x ] _cumsum_type{T<:Number}(v::AbstractArray{T}) = typeof(+zero(T)) _cumsum_type(v) = typeof(v[1]+v[1]) diff --git a/test/offsetarray.jl b/test/offsetarray.jl index 2f26454826c7a..d7ebf5a66fd6e 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -101,6 +101,12 @@ using OAs let # Basics +v0 = rand(4) +v = OffsetArray(v0, (-3,)) +@test indices(v) == (-2:1,) +@test_throws ErrorException size(v) +@test_throws ErrorException size(v, 1) + A0 = [1 3; 2 4] A = OffsetArray(A0, (-1,2)) # LinearFast S = OffsetArray(view(A0, 1:2, 1:2), (-1,2)) # LinearSlow @@ -179,6 +185,10 @@ end # show io = IOBuffer() +show(io, v) +str = takebuf_string(io) +show(io, v0) +@test str == takebuf_string(io) show(io, A) str = takebuf_string(io) @test str == "[1 3; 2 4]" @@ -261,7 +271,7 @@ v = view(A0, 1:1, i1) # logical indexing @test A[A .> 2] == [3,4] -# copy! +# copy! and fill! a = OffsetArray{Int}((-3:-1,)) fill!(a, -1) copy!(a, (1,2)) # non-array iterables @@ -316,6 +326,7 @@ copy!(am, b) @test am[1,8] == 2 @test am[1,9] == -1 +# map dest = similar(am) map!(+, dest, am, am) @test dest[1,7] == 2 @@ -326,6 +337,10 @@ am = map(identity, a) @test isa(am, OffsetArray) @test am == a +# other functions +v = OffsetArray(v0, (-3,)) +@test parent(v') == v0' +@test indices(v') === (1:1,-2:1) A = OffsetArray(rand(4,4), (-3,5)) @test maximum(A) == maximum(parent(A)) @test minimum(A) == minimum(parent(A)) From bee5a347bb0f01aa154589fd8521e07c29606e36 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Thu, 4 Aug 2016 07:30:44 -0500 Subject: [PATCH 11/80] Get atsign-test_approx_eq working for non-1 indices (cherry picked from commit 06c6d59d9228c4a571e29f2cb0bf659f8f0e204f) ref #17816 --- base/test.jl | 15 ++++++++------- test/offsetarray.jl | 2 ++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/base/test.jl b/base/test.jl index 586f5767906ef..7d23115354bbd 100644 --- a/base/test.jl +++ b/base/test.jl @@ -827,10 +827,11 @@ approx_full(x) = full(x) function test_approx_eq(va, vb, Eps, astr, bstr) va = approx_full(va) vb = approx_full(vb) - if length(va) != length(vb) + la, lb = length(linearindices(va)), length(linearindices(vb)) + if la != lb error("lengths of ", astr, " and ", bstr, " do not match: ", - "\n ", astr, " (length $(length(va))) = ", va, - "\n ", bstr, " (length $(length(vb))) = ", vb) + "\n ", astr, " (length $la) = ", va, + "\n ", bstr, " (length $lb) = ", vb) end diff = real(zero(eltype(va))) for (xa, xb) = zip(va, vb) @@ -856,7 +857,7 @@ array_eps{T}(a::AbstractArray{Complex{T}}) = eps(float(maximum(x->(isfinite(x) ? array_eps(a) = eps(float(maximum(x->(isfinite(x) ? abs(x) : oftype(x,NaN)), a))) test_approx_eq(va, vb, astr, bstr) = - test_approx_eq(va, vb, 1E4*length(va)*max(array_eps(va), array_eps(vb)), astr, bstr) + test_approx_eq(va, vb, 1E4*length(linearindices(va))*max(array_eps(va), array_eps(vb)), astr, bstr) """ @test_approx_eq_eps(a, b, tol) @@ -966,10 +967,10 @@ end # nothing. function test_approx_eq_modphase{S<:Real,T<:Real}( a::StridedVecOrMat{S}, b::StridedVecOrMat{T}, err=nothing) - m, n = size(a) - @test n==size(b, 2) && m==size(b, 1) + @test indices(a,1) == indices(b,1) && indices(a,2) == indices(b,2) + m = length(indices(a,1)) err === nothing && (err=m^3*(eps(S)+eps(T))) - for i=1:n + for i in indices(a,2) v1, v2 = a[:, i], b[:, i] @test_approx_eq_eps min(abs(norm(v1-v2)), abs(norm(v1+v2))) 0.0 err end diff --git a/test/offsetarray.jl b/test/offsetarray.jl index d7ebf5a66fd6e..4289aa523fe17 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -339,9 +339,11 @@ am = map(identity, a) # other functions v = OffsetArray(v0, (-3,)) +@test_approx_eq v v @test parent(v') == v0' @test indices(v') === (1:1,-2:1) A = OffsetArray(rand(4,4), (-3,5)) +@test_approx_eq A A @test maximum(A) == maximum(parent(A)) @test minimum(A) == minimum(parent(A)) @test extrema(A) == extrema(parent(A)) From 3d980ca2db5a984cc093f0d628d96c8cec3bdf08 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Thu, 4 Aug 2016 07:35:57 -0500 Subject: [PATCH 12/80] Generalize findn and findnz for non-1 indices (cherry picked from commit 37e9c5cb2d583d8b44ea31c855a93574b42c5c8a) ref #17816 --- base/array.jl | 4 ++-- test/offsetarray.jl | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/base/array.jl b/base/array.jl index f20d5ccd1a5cb..783b93e0e5956 100644 --- a/base/array.jl +++ b/base/array.jl @@ -795,7 +795,7 @@ function findn(A::AbstractMatrix) I = similar(A, Int, nnzA) J = similar(A, Int, nnzA) count = 1 - for j=1:size(A,2), i=1:size(A,1) + for j=indices(A,2), i=indices(A,1) if A[i,j] != 0 I[count] = i J[count] = j @@ -812,7 +812,7 @@ function findnz{T}(A::AbstractMatrix{T}) NZs = Array{T,1}(nnzA) count = 1 if nnzA > 0 - for j=1:size(A,2), i=1:size(A,1) + for j=indices(A,2), i=indices(A,1) Aij = A[i,j] if Aij != 0 I[count] = i diff --git a/test/offsetarray.jl b/test/offsetarray.jl index 4289aa523fe17..5ca538539ec6a 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -369,6 +369,14 @@ pmax, ipmax = findmax(parent(A)) @test amax == pmax @test A[iamax] == amax @test amax == parent(A)[ipmax] +z = OffsetArray([0 0; 2 0; 0 0; 0 0], (-3,-1)) +I,J = findn(z) +@test I == [-1] +@test J == [0] +I,J,N = findnz(z) +@test I == [-1] +@test J == [0] +@test N == [2] v = OffsetArray([1,1e100,1,-1e100], (-3,))*1000 v2 = OffsetArray([1,-1e100,1,1e100], (5,))*1000 From 631ff0088bf41875d7a2433fbf189d5eaab48d84 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Thu, 4 Aug 2016 08:24:30 +0800 Subject: [PATCH 13/80] Deoptimize TLS access * Apparently `ifunc` is not supported by gcc 5 on ARM * Try even harder to workaround LLVM bug when dealing with returntwice function (cherry picked from commit f90eab3c4a00193b6c4fb1d0d8994afbdb540457) ref #17800 --- src/llvm-ptls.cpp | 45 ++++++++++++++++++++++++++++++++++----------- src/threading.c | 2 +- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/src/llvm-ptls.cpp b/src/llvm-ptls.cpp index 178faf6c3b28e..019975cc3ba05 100644 --- a/src/llvm-ptls.cpp +++ b/src/llvm-ptls.cpp @@ -7,6 +7,7 @@ #include "llvm-version.h" #include "support/dtypes.h" +#include #include #include @@ -99,23 +100,41 @@ void LowerPTLS::runOnFunction(LLVMContext &ctx, Module &M, Function *F, ptlsStates->addAttribute(AttributeSet::FunctionIndex, Attribute::NoUnwind); } - else if (jl_tls_offset != -1) { #ifdef LLVM37 + else if (jl_tls_offset != -1) { auto T_int8 = Type::getInt8Ty(ctx); auto T_pint8 = PointerType::get(T_int8, 0); - auto T_size = (sizeof(size_t) == 8 ? Type::getInt64Ty(ctx) : - Type::getInt32Ty(ctx)); // Replace the function call with inline assembly if we know // how to generate it. - const char *asm_str = nullptr; +# if defined(_CPU_X86_64_) || defined(_CPU_X86_) + // Workaround LLVM bug by hiding the offset computation + // (and therefore the optimization opportunity) from LLVM. + static const std::string asm_str = [&] () { + std::stringstream stm; # if defined(_CPU_X86_64_) - asm_str = "movq %fs:0, $0"; -# elif defined(_CPU_X86_) - asm_str = "movl %gs:0, $0"; -# elif defined(_CPU_AARCH64_) - asm_str = "mrs $0, tpidr_el0"; + stm << "movq %fs:0, $0;\naddq $$" << jl_tls_offset << ", $0"; +# else + stm << "movl %gs:0, $0;\naddl $$" << jl_tls_offset << ", $0"; # endif - assert(asm_str && "Cannot emit thread pointer for this architecture."); + return stm.str(); + }(); + // The add instruction clobbers flags + auto tp = InlineAsm::get(FunctionType::get(T_pint8, false), + asm_str.c_str(), + "=r,~{dirflag},~{fpsr},~{flags}", false); + Value *tls = CallInst::Create(tp, "ptls_i8", ptlsStates); + tls = new BitCastInst(tls, PointerType::get(T_ppjlvalue, 0), + "ptls", ptlsStates); +# elif defined(_CPU_AARCH64_) + // AArch64 doesn't seem to have this issue. + // (Possibly because there are many more registers and the offset is + // positive and small) + // It's also harder to emit the offset in a generic way on AArch64 + // (need to generate one or two `add` with shift) so let llvm emit + // the add for now. + auto T_size = (sizeof(size_t) == 8 ? Type::getInt64Ty(ctx) : + Type::getInt32Ty(ctx)); + const char *asm_str = "mrs $0, tpidr_el0"; auto offset = ConstantInt::getSigned(T_size, jl_tls_offset); auto tp = InlineAsm::get(FunctionType::get(T_pint8, false), asm_str, "=r", false); @@ -124,10 +143,14 @@ void LowerPTLS::runOnFunction(LLVMContext &ctx, Module &M, Function *F, "ptls_i8", ptlsStates); tls = new BitCastInst(tls, PointerType::get(T_ppjlvalue, 0), "ptls", ptlsStates); +# else + Value *tls = nullptr; + assert(0 && "Cannot emit thread pointer for this architecture."); +# endif ptlsStates->replaceAllUsesWith(tls); ptlsStates->eraseFromParent(); -#endif } +#endif else { ptlsStates->addAttribute(AttributeSet::FunctionIndex, Attribute::ReadNone); diff --git a/src/threading.c b/src/threading.c index b4de746abe526..6b1d373c7f1f6 100644 --- a/src/threading.c +++ b/src/threading.c @@ -147,7 +147,7 @@ jl_get_ptls_states_func jl_get_ptls_states_getter(void) #if defined(__GLIBC__) && (defined(_CPU_X86_64_) || defined(_CPU_X86_) || \ ((defined(_CPU_AARCH64_) || defined(_CPU_ARM_) || \ defined(_CPU_PPC64_) || defined(_CPU_PPC_)) && \ - __GNUC__ >= 5)) + __GNUC__ >= 6)) // Only enable this on architectures that are tested. // For example, GCC doesn't seem to support the `ifunc` attribute on power yet. # if __GLIBC_PREREQ(2, 12) From d594e10032e19045cc6191b92a0943a29816f348 Mon Sep 17 00:00:00 2001 From: kshyatt Date: Fri, 5 Aug 2016 10:08:22 -0700 Subject: [PATCH 14/80] Move docs out of helpDB, make them less verbose. (cherry picked from commit 0aac966038d09cfa59a94ab002c655210be7c1be) ref #17841 --- base/docs/helpdb/Base.jl | 144 --------------------------------------- base/special/gamma.jl | 40 +++++++++++ base/special/trig.jl | 36 ++++++++-- 3 files changed, 71 insertions(+), 149 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 9a08432feec8e..13b8490384b9b 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -22,13 +22,6 @@ tab-delimited text to `f` by either `writedlm(f, [x y])` or by `writedlm(f, zip( """ writedlm -""" - digamma(x) - -Compute the digamma function of `x` (the logarithmic derivative of `gamma(x)`) -""" -digamma - """ fill!(A, x) @@ -523,13 +516,6 @@ A string giving the literal bit representation of a number. """ bits -""" - invdigamma(x) - -Compute the inverse digamma function of `x`. -""" -invdigamma - """ getindex(type[, elements...]) @@ -609,13 +595,6 @@ used as the final delimiter instead of `delim`. """ join(io, items, delim, last) -""" - lfact(x) - -Compute the logarithmic factorial of `x` -""" -lfact - """ deconv(b,a) @@ -1007,15 +986,6 @@ Test whether a matrix is lower triangular. """ istril -""" - lgamma(x) - -Compute the logarithm of the absolute value of [`gamma`](:func:`gamma`) for -[`Real`](:obj:`Real`) `x`, while for [`Complex`](:obj:`Complex`) `x` it computes the -logarithm of `gamma(x)`. -""" -lgamma - """ bin(n, [pad]) @@ -1445,13 +1415,6 @@ Close an I/O stream. Performs a `flush` first. """ close(stream::IO) -""" - cospi(x) - -Compute ``\\cos(\\pi x)`` more accurately than `cos(pi*x)`, especially for large `x`. -""" -cospi - """ parentindexes(A) @@ -2420,13 +2383,6 @@ Bessel function of the first kind of order 1, ``J_1(x)``. """ besselj1 -""" - sinpi(x) - -Compute ``\\sin(\\pi x)`` more accurately than `sin(pi*x)`, especially for large `x`. -""" -sinpi - """ select!(v, k, [by=,] [lt=,] [rev=false]) @@ -2872,13 +2828,6 @@ Seek a stream to the given position. """ seek -""" - acosd(x) - -Compute the inverse cosine of `x`, where the output is in degrees. -""" -acosd - """ triu(M) @@ -3013,13 +2962,6 @@ Equivalent to `stat(file).size`. """ filesize -""" - sinc(x) - -Compute ``\\sin(\\pi x) / (\\pi x)`` if ``x \\neq 0``, and ``1`` if ``x = 0``. -""" -sinc - """ cglobal((symbol, library) [, type=Void]) @@ -3650,13 +3592,6 @@ Returns the smallest eigenvalue of `A`. """ eigmin -""" - acscd(x) - -Compute the inverse cosecant of `x`, where the output is in degrees. -""" -acscd - """ ltoh(x) @@ -4142,13 +4077,6 @@ is `-1` the corresponding ID will not change. Only integer `owner`s and `group`s """ chown -""" - gamma(x) - -Compute the gamma function of `x`. -""" -gamma - """ sin(x) @@ -4855,13 +4783,6 @@ For more information, see [^issue8859], [^B96], [^S84], [^KY88]. """ pinv -""" - asecd(x) - -Compute the inverse secant of `x`, where the output is in degrees. -""" -asecd - """ readbytes!(stream::IO, b::AbstractVector{UInt8}, nb=length(b); all=true) @@ -4888,13 +4809,6 @@ descriptive error string. """ ArgumentError -""" - atand(x) - -Compute the inverse tangent of `x`, where the output is in degrees. -""" -atand - """ KeyError(key) @@ -5062,13 +4976,6 @@ Test whether a matrix is Hermitian. """ ishermitian -""" - sind(x) - -Compute sine of `x`, where `x` is in degrees. -""" -sind - """ min(x, y, ...) @@ -5380,14 +5287,6 @@ two strings. For example """ join(strings, delim, last) -""" - polygamma(m, x) - -Compute the polygamma function of order `m` of argument `x` (the `(m+1)th` derivative of the -logarithm of `gamma(x)`) -""" -polygamma - """ isless(x, y) @@ -6055,13 +5954,6 @@ Returns `string` with all characters converted to uppercase. """ uppercase -""" - cosd(x) - -Compute cosine of `x`, where `x` is in degrees. -""" -cosd - """ cycle(iter) @@ -6500,13 +6392,6 @@ lengths of dimensions you asked for. """ size -""" - trigamma(x) - -Compute the trigamma function of `x` (the logarithmic second derivative of `gamma(x)`). -""" -trigamma - """ findmin(A, dims) -> (minval, index) @@ -7188,13 +7073,6 @@ but throws an error for unordered arguments. """ cmp -""" - tand(x) - -Compute tangent of `x`, where `x` is in degrees. -""" -tand - """ issorted(v, [by=,] [lt=,] [rev=false]) @@ -7583,14 +7461,6 @@ Test whether any values along the given dimensions of an array are `true`. """ any(::AbstractArray,dims) -""" - cosc(x) - -Compute ``\\cos(\\pi x) / x - \\sin(\\pi x) / (\\pi x^2)`` if ``x \\neq 0``, and ``0`` if -``x = 0``. This is the derivative of `sinc(x)`. -""" -cosc - """ getkey(collection, key, default) @@ -7612,13 +7482,6 @@ For matrices or vectors ``A`` and ``B``, calculates ``Aᴴ Bᴴ``. """ Ac_mul_Bc -""" - acotd(x) - -Compute the inverse cotangent of `x`, where the output is in degrees. -""" -acotd - """ zeros(type, dims) @@ -8313,13 +8176,6 @@ julia> f(apple) """ :@enum -""" - asind(x) - -Compute the inverse sine of `x`, where the output is in degrees. -""" -asind - """ widemul(x, y) diff --git a/base/special/gamma.jl b/base/special/gamma.jl index e1bbe630f1e33..083ad14c30fdb 100644 --- a/base/special/gamma.jl +++ b/base/special/gamma.jl @@ -2,6 +2,12 @@ gamma(x::Float64) = nan_dom_err(ccall((:tgamma,libm), Float64, (Float64,), x), x) gamma(x::Float32) = nan_dom_err(ccall((:tgammaf,libm), Float32, (Float32,), x), x) + +""" + gamma(x) + +Compute the gamma function of `x`. +""" gamma(x::Real) = gamma(float(x)) @vectorize_1arg Number gamma @@ -19,6 +25,11 @@ lgamma_r(x::Real) = lgamma_r(float(x)) lgamma_r(x::Number) = lgamma(x), 1 # lgamma does not take abs for non-real x "`lgamma_r(x)`: return L,s such that `gamma(x) = s * exp(L)`" lgamma_r +""" + lfact(x) + +Compute the logarithmic factorial of `x` +""" lfact(x::Real) = (x<=1 ? zero(float(x)) : lgamma(x+one(x))) @vectorize_1arg Number lfact @@ -47,6 +58,13 @@ function clgamma_lanczos(z) return log(zz) - temp end +""" + lgamma(x) + +Compute the logarithm of the absolute value of [`gamma`](:func:`gamma`) for +[`Real`](:obj:`Real`) `x`, while for [`Complex`](:obj:`Complex`) `x` it computes the +logarithm of `gamma(x)`. +""" function lgamma(z::Complex) if real(z) <= 0.5 a = clgamma_lanczos(1-z) @@ -69,6 +87,11 @@ gamma(z::Complex) = exp(lgamma(z)) # const A002445 = [1,6,30,42,30,66,2730,6,510,798,330,138,2730,6,870,14322,510,6,1919190,6,13530] # const bernoulli = A000367 .// A002445 # even-index Bernoulli numbers +""" + digamma(x) + +Compute the digamma function of `x` (the logarithmic derivative of `gamma(x)`) +""" function digamma(z::Union{Float64,Complex{Float64}}) # Based on eq. (12), without looking at the accompanying source # code, of: K. S. Kölbig, "Programs for computing the logarithm of @@ -98,6 +121,11 @@ function digamma(z::Union{Float64,Complex{Float64}}) ψ -= t * @evalpoly(t,0.08333333333333333,-0.008333333333333333,0.003968253968253968,-0.004166666666666667,0.007575757575757576,-0.021092796092796094,0.08333333333333333,-0.4432598039215686) end +""" + trigamma(x) + +Compute the trigamma function of `x` (the logarithmic second derivative of `gamma(x)`). +""" function trigamma(z::Union{Float64,Complex{Float64}}) # via the derivative of the Kölbig digamma formulation x = real(z) @@ -360,6 +388,12 @@ function zeta(s::Union{Int,Float64,Complex{Float64}}, return ζ end +""" + polygamma(m, x) + +Compute the polygamma function of order `m` of argument `x` (the `(m+1)th` derivative of the +logarithm of `gamma(x)`) +""" function polygamma(m::Integer, z::Union{Float64,Complex{Float64}}) m == 0 && return digamma(z) m == 1 && return trigamma(z) @@ -445,6 +479,12 @@ function invdigamma(y::Float64) return x_new end invdigamma(x::Float32) = Float32(invdigamma(Float64(x))) + +""" + invdigamma(x) + +Compute the inverse digamma function of `x`. +""" invdigamma(x::Real) = invdigamma(Float64(x)) @vectorize_1arg Real invdigamma diff --git a/base/special/trig.jl b/base/special/trig.jl index ab1ac1b4efafe..087f2ad0338e6 100644 --- a/base/special/trig.jl +++ b/base/special/trig.jl @@ -98,7 +98,11 @@ mulpi_ext(x::Float32) = DoubleFloat32(pi*Float64(x)) mulpi_ext(x::Rational) = mulpi_ext(float(x)) mulpi_ext(x::Real) = pi*x # Fallback +""" + sinpi(x) +Compute ``\\sin(\\pi x)`` more accurately than `sin(pi*x)`, especially for large `x`. +""" function sinpi{T<:AbstractFloat}(x::T) if !isfinite(x) isnan(x) && return x @@ -157,6 +161,11 @@ function sinpi{T<:Real}(x::T) end end +""" + cospi(x) + +Compute ``\\cos(\\pi x)`` more accurately than `cos(pi*x)`, especially for large `x`. +""" function cospi{T<:AbstractFloat}(x::T) if !isfinite(x) isnan(x) && return x @@ -278,13 +287,23 @@ end @vectorize_1arg Number sinpi @vectorize_1arg Number cospi +""" + sinc(x) +Compute ``\\sin(\\pi x) / (\\pi x)`` if ``x \\neq 0``, and ``1`` if ``x = 0``. +""" sinc(x::Number) = x==0 ? one(x) : oftype(x,sinpi(x)/(pi*x)) sinc(x::Integer) = x==0 ? one(x) : zero(x) sinc{T<:Integer}(x::Complex{T}) = sinc(float(x)) sinc(x::Real) = x==0 ? one(x) : isinf(x) ? zero(x) : sinpi(x)/(pi*x) @vectorize_1arg Number sinc +""" + cosc(x) + +Compute ``\\cos(\\pi x) / x - \\sin(\\pi x) / (\\pi x^2)`` if ``x \\neq 0``, and ``0`` if +``x = 0``. This is the derivative of `sinc(x)`. +""" cosc(x::Number) = x==0 ? zero(x) : oftype(x,(cospi(x)-sinpi(x)/(pi*x))/x) cosc(x::Integer) = cosc(float(x)) cosc{T<:Integer}(x::Complex{T}) = cosc(float(x)) @@ -389,16 +408,23 @@ end tand(x::Real) = sind(x) / cosd(x) @vectorize_1arg Real tand -for (fd, f) in ((:sind, :sin), (:cosd, :cos), (:tand, :tan)) +for (fd, f, fn) in ((:sind, :sin, "sine"), (:cosd, :cos, "cosine"), (:tand, :tan, "tangent")) + name = string(fd) @eval begin - ($fd)(z) = ($f)(deg2rad(z)) + @doc """ + $($name)(x) + Compute $($fn) of `x`, where `x` is in degrees. """ ($fd)(z) = ($f)(deg2rad(z)) end end -for (fd, f) in ((:asind, :asin), (:acosd, :acos), (:atand, :atan), - (:asecd, :asec), (:acscd, :acsc), (:acotd, :acot)) +for (fd, f, fn) in ((:asind, :asin, "sine"), (:acosd, :acos, "cosine"), (:atand, :atan, "tangent"), + (:asecd, :asec, "secant"), (:acscd, :acsc, "cosecant"), (:acotd, :acot, "cotangent")) + name = string(fd) @eval begin - ($fd)(y) = rad2deg(($f)(y)) + @doc """ + $($name)(x) + + Compute the inverse $($fn) of `x`, where the output is in degrees. """ ($fd)(y) = rad2deg(($f)(y)) @vectorize_1arg Real $fd end end From 0bbe0276807780dc964e5e43a1c5b52525ce5a29 Mon Sep 17 00:00:00 2001 From: WooKyoung Noh Date: Fri, 5 Aug 2016 20:08:19 +0900 Subject: [PATCH 15/80] Add tests for date parsing (#10817) (cherry picked from commit f2863efb5a52800a65be73b2377b717212fab0c4) ref #17836 --- test/dates/io.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/dates/io.jl b/test/dates/io.jl index 11aea94137a1b..bdcade050c16f 100644 --- a/test/dates/io.jl +++ b/test/dates/io.jl @@ -359,3 +359,8 @@ let @test DateTime(ds, format) == dt @test DateTime(ds, escaped_format) == dt end + +# Issue 10817 +@test Dates.Date("Apr 01 2014", "uuu dd yyyy") == Dates.Date(2014,4,1) +@test_throws ArgumentError Dates.Date("Apr 01 xx 2014", "uuu dd zz yyyy") +@test_throws ArgumentError Dates.Date("Apr 01 xx 2014", "uuu dd yyyy") From df8dcf3fc0634019e083ba2b85fe081fd08cc914 Mon Sep 17 00:00:00 2001 From: Tracy Wadleigh Date: Fri, 5 Aug 2016 15:27:49 -0700 Subject: [PATCH 16/80] Recognize atom with a file extension. The `edit` function on windows fails if the editor command is `atom`. It succeeds if it is instead `atom.cmd`, but then Julia fails to recognize the `atom` editor. This fixes that. (cherry picked from commit e79bee701653c76eec6d01328fa6cfec43fd1493) ref #17857 --- base/interactiveutil.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/interactiveutil.jl b/base/interactiveutil.jl index 0a757ee10b619..7ae43796fedf0 100644 --- a/base/interactiveutil.jl +++ b/base/interactiveutil.jl @@ -40,7 +40,7 @@ function edit(path::AbstractString, line::Integer=0) background = false elseif name == "textmate" || name == "mate" || name == "kate" cmd = line != 0 ? `$command $path -l $line` : `$command $path` - elseif startswith(name, "subl") || name == "atom" + elseif startswith(name, "subl") || startswith(name, "atom") cmd = line != 0 ? `$command $path:$line` : `$command $path` elseif is_windows() && (name == "start" || name == "open") cmd = `cmd /c start /b $path` From a403819e0a4781ad3c87e23d34cae90752a26058 Mon Sep 17 00:00:00 2001 From: Spencer Russell Date: Fri, 5 Aug 2016 18:14:42 -0400 Subject: [PATCH 17/80] AST doc - comparisons now use `call` unless chained (cherry picked from commit 008b15fef41ededf255a7948034178414c993621) ref #17855 --- doc/devdocs/ast.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/devdocs/ast.rst b/doc/devdocs/ast.rst index 7cfe51bb4ac18..72712803a5fcc 100644 --- a/doc/devdocs/ast.rst +++ b/doc/devdocs/ast.rst @@ -267,7 +267,7 @@ Input AST ``a:b`` ``(: a b)`` ``a:b:c`` ``(: a b c)`` ``a,b`` ``(tuple a b)`` -``a==b`` ``(comparison a == b)`` +``a==b`` ``(call == a b)`` ``1 Date: Fri, 5 Aug 2016 16:00:28 -0400 Subject: [PATCH 18/80] Clarifies location of Sublime Text editing mode It is now in a separate repo and not in `contrib/` (cherry picked from commit c367b42b30e3a370b0d1eaed73d93368e7a44d37) ref #17849 --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 87aa47027db59..6baaf4c2343a1 100644 --- a/README.md +++ b/README.md @@ -418,9 +418,10 @@ The following distributions include julia, but the versions may be out of date d Currently, Julia editing mode support is available for a number of editors. While Julia modes for -[Emacs](https://github.com/JuliaLang/julia-emacs) and +[Emacs](https://github.com/JuliaLang/julia-emacs), +[Sublime Text](https://github.com/JuliaEditorSupport/Julia-sublime), and [Vim](https://github.com/JuliaLang/julia-vim) have their own repos, -others such as Textmate, Sublime Text, Notepad++, and Kate, are in +others such as Textmate, Notepad++, and Kate, are in `contrib/`. Two major IDEs are supported for Julia: [Juno](http://junolab.org/), From ea617ec8595ca678f3f69f9182fe33d2da7d4094 Mon Sep 17 00:00:00 2001 From: kshyatt Date: Wed, 3 Aug 2016 16:32:33 -0700 Subject: [PATCH 19/80] More HelpDB pruning for Bit/SparseArray, more examples Yet MORE examples from the REPL. Fixed and added links, changd some wording. (cherry picked from commit e4a377caada1ca49c85683fee04324987d3edbcf) ref #17797 --- base/bitarray.jl | 134 +++++++++++++++++++++++++++ base/docs/helpdb/Base.jl | 49 ---------- base/sparse/sparsematrix.jl | 81 +++++++++++++++-- doc/stdlib/arrays.rst | 177 +++++++++++++++++++++++++++++++++--- 4 files changed, 368 insertions(+), 73 deletions(-) diff --git a/base/bitarray.jl b/base/bitarray.jl index f3e7ff9c90689..95115cb3501d9 100644 --- a/base/bitarray.jl +++ b/base/bitarray.jl @@ -1005,6 +1005,25 @@ function (~)(B::BitArray) return C end +""" + flipbits!(B::BitArray{N}) -> BitArray{N} + +Performs a bitwise not operation on `B`. See [`~`](:ref:`~ operator <~>`). + +```jldoctest +julia> A = bitrand(3,5) +3×5 BitArray{2}: + true true true true true + false false false true false + true false false false true + +julia> flipbits!(A) +3×5 BitArray{2}: + false false false false false + true true true false true + false true true true false +``` +""" function flipbits!(B::BitArray) Bc = B.chunks @inbounds if !isempty(Bc) @@ -1405,6 +1424,39 @@ end (>>)(B::BitVector, i::Int) = B >>> i +""" + rol!(dest::BitVector, src::BitVector, i::Integer) -> BitVector + +Performs a left rotation operation on `src` and put the result into `dest`. + +```jldoctest +julia> a = bitrand(5) +5-element BitArray{1}: + false + false + true + true + true + +julia> b = falses(5); + +julia> rol!(b,a,2) +5-element BitArray{1}: + true + true + true + false + false + +julia> rol!(b,a,3) +5-element BitArray{1}: + true + true + false + false + true +``` +""" function rol!(dest::BitVector, src::BitVector, i::Integer) length(dest) == length(src) || throw(ArgumentError("destination and source should be of same size")) n = length(dest) @@ -1417,14 +1469,67 @@ function rol!(dest::BitVector, src::BitVector, i::Integer) return dest end +""" + rol!(B::BitVector, i::Integer) -> BitVector + +Performs a left rotation operation on `B`. + +```jldoctest +julia> a = bitrand(5) +5-element BitArray{1}: + true + false + false + true + true + +julia> rol!(a,4) +5-element BitArray{1}: + true + true + false + false + true +""" function rol!(B::BitVector, i::Integer) return rol!(B, B, i) end +""" + rol(B::BitVector, i::Integer) -> BitVector + +Performs a left rotation operation without modifying `B`. +See also [`rol!`](:func:`rol!`). +""" function rol(B::BitVector, i::Integer) return rol!(similar(B), B, i) end +""" + ror!(dest::BitVector, src::BitVector, i::Integer) -> BitVector + +Performs a right rotation operation on `src` and put the result into `dest`. + +```jldoctest +julia> a = bitrand(5) +5-element BitArray{1}: + false + false + true + true + true + +julia> b = falses(5); + +julia> ror!(b,a,2) +5-element BitArray{1}: + true + true + false + false + true +``` +""" function ror!(dest::BitVector, src::BitVector, i::Integer) length(dest) == length(src) || throw(ArgumentError("destination and source should be of same size")) n = length(dest) @@ -1437,10 +1542,39 @@ function ror!(dest::BitVector, src::BitVector, i::Integer) return dest end +""" + ror!(B::BitVector, i::Integer) -> BitVector + +Performs a right rotation operation on `B`. + +```jldoctest +julia> a = bitrand(5) +5-element BitArray{1}: + false + true + false + true + false + +julia> ror!(a,3) +5-element BitArray{1}: + false + true + false + false + true +``` +""" function ror!(B::BitVector, i::Integer) return ror!(B, B, i) end +""" + ror(B::BitVector, i::Integer) -> BitVector + +Performs a right rotation operation without modifying `B`. +See also [`ror!`](:func:`ror!`). +""" function ror(B::BitVector, i::Integer) return ror!(similar(B), B, i) end diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 13b8490384b9b..51855d833c1db 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -393,20 +393,6 @@ Element-wise multiplication operator. """ Base.:(.*) -""" - ror!(dest::BitArray{1}, src::BitArray{1}, i::Integer) -> BitArray{1} - -Performs a right rotation operation on `src` and put the result into `dest`. -""" -ror!(dest::BitArray{1}, src::BitArray{1}, i::Integer) - -""" - ror!(B::BitArray{1}, i::Integer) -> BitArray{1} - -Performs a right rotation operation on `B`. -""" -ror!(B::BitArray{1}, i::Integer) - """ range(start, [step], length) @@ -784,13 +770,6 @@ Returns `true` if `path` has the sticky bit set, `false` otherwise. """ issticky -""" - rol(B::BitArray{1}, i::Integer) -> BitArray{1} - -Performs a left rotation operation. -""" -rol - """ Mmap.mmap(io::Union{IOStream,AbstractString,Mmap.AnonymousMmap}[, type::Type{Array{T,N}}, dims, offset]; grow::Bool=true, shared::Bool=true) Mmap.mmap(type::Type{Array{T,N}}, dims) @@ -4258,13 +4237,6 @@ Returns `string` with the first character converted to lowercase. """ lcfirst -""" - flipbits!(B::BitArray{N}) -> BitArray{N} - -Performs a bitwise not operation on `B`. See [`~`](:ref:`~ operator <~>`). -""" -flipbits! - """ readlink(path) -> AbstractString @@ -7343,20 +7315,6 @@ dims)` will return an array filled with the result of evaluating `Foo()` once. """ fill -""" - rol!(dest::BitArray{1}, src::BitArray{1}, i::Integer) -> BitArray{1} - -Performs a left rotation operation on `src` and put the result into `dest`. -""" -rol!(::BitArray,::BitArray,::Integer) - -""" - rol!(B::BitArray{1}, i::Integer) -> BitArray{1} - -Performs a left rotation operation on `B`. -""" -rol!(::BitArray,::Integer) - """ issubset(a, b) ⊆(a,b) -> Bool @@ -8143,13 +8101,6 @@ Optional argument `msg` is a descriptive error string. """ AssertionError -""" - ror(B::BitArray{1}, i::Integer) -> BitArray{1} - -Performs a right rotation operation. -""" -ror - """ Ac_ldiv_Bc(A, B) diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index 821e04d4d18da..fa25fa2e64ac4 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -40,8 +40,8 @@ countnz(S::SparseMatrixCSC) = countnz(S.nzval) Return a vector of the structural nonzero values in sparse array `A`. This includes zeros that are explicitly stored in the sparse array. The returned vector points directly to the internal nonzero storage of `A`, and any -modifications to the returned vector will mutate `A` as well. See `rowvals(A)` -and `nzrange(A, col)`. +modifications to the returned vector will mutate `A` as well. See +[`rowvals`](:func:`rowvals`) and [`nzrange`](:func:`nzrange`). """ nonzeros(S::SparseMatrixCSC) = S.nzval @@ -51,7 +51,7 @@ nonzeros(S::SparseMatrixCSC) = S.nzval Return a vector of the row indices of `A`. Any modifications to the returned vector will mutate `A` as well. Providing access to how the row indices are stored internally can be useful in conjunction with iterating over structural -nonzero values. See also `nonzeros(A)` and `nzrange(A, col)`. +nonzero values. See also [`nonzeros`](:func:`nonzeros`) and [`nzrange`](:func:`nzrange`). """ rowvals(S::SparseMatrixCSC) = S.rowval @@ -59,8 +59,8 @@ rowvals(S::SparseMatrixCSC) = S.rowval nzrange(A::SparseMatrixCSC, col) Return the range of indices to the structural nonzero values of a sparse matrix -column. In conjunction with `nonzeros(A)` and `rowvals(A)`, this allows for -convenient iterating over a sparse matrix : +column. In conjunction with [`nonzeros`](:func:`nonzeros`) and +[`rowvals`](:func:`rowvals`), this allows for convenient iterating over a sparse matrix : A = sparse(I,J,V) rows = rowvals(A) @@ -374,7 +374,7 @@ are set to `maximum(I)` and `maximum(J)` respectively. If the `combine` function supplied, `combine` defaults to `+` unless the elements of `V` are Booleans in which case `combine` defaults to `|`. All elements of `I` must satisfy `1 <= I[k] <= m`, and all elements of `J` must satisfy `1 <= J[k] <= n`. Numerical zeros in (`I`, `J`, `V`) are -retained as structural nonzeros; to drop numerical zeros, use `dropzeros!`. +retained as structural nonzeros; to drop numerical zeros, use [`dropzeros!`](:func:`dropzeros!`). For additional documentation and an expert driver, see `Base.SparseArrays.sparse!`. """ @@ -1044,7 +1044,7 @@ droptol!(A::SparseMatrixCSC, tol, trim::Bool = true) = Removes stored numerical zeros from `A`, optionally trimming resulting excess space from `A.rowval` and `A.nzval` when `trim` is `true`. -For an out-of-place version, see [`dropzeros`](:func:`Base.SparseArrays.dropzeros`). For +For an out-of-place version, see [`dropzeros`](:func:`dropzeros`). For algorithmic information, see [`Base.SparseArrays.fkeep!`](:func:`Base.SparseArrays.fkeep!`). """ dropzeros!(A::SparseMatrixCSC, trim::Bool = true) = fkeep!(A, (i, j, x) -> x != 0, trim) @@ -1054,7 +1054,7 @@ dropzeros!(A::SparseMatrixCSC, trim::Bool = true) = fkeep!(A, (i, j, x) -> x != Generates a copy of `A` and removes stored numerical zeros from that copy, optionally trimming excess space from the result's `rowval` and `nzval` arrays when `trim` is `true`. -For an in-place version and algorithmic information, see [`dropzeros!`](:func:`Base.SparseArrays.dropzeros!`). +For an in-place version and algorithmic information, see [`dropzeros!`](:func:`dropzeros!`). """ dropzeros(A::SparseMatrixCSC, trim::Bool = true) = dropzeros!(copy(A), trim) @@ -1212,6 +1212,32 @@ sprandn(m::Integer, n::Integer, density::AbstractFloat) = sprandn(GLOBAL_RNG,m,n Create a sparse array with the same structure as that of `S`, but with every nonzero element having the value `1.0`. + +```jldoctest +julia> A = sprand(5,6,0.2) +5×6 sparse matrix with 8 Float64 nonzero entries: + [2, 1] = 0.639431 + [5, 1] = 0.881209 + [3, 2] = 0.355834 + [4, 2] = 0.904768 + [2, 3] = 0.760943 + [3, 5] = 0.525942 + [4, 5] = 0.936283 + [5, 5] = 0.432364 + +julia> spones(A) +5×6 sparse matrix with 8 Float64 nonzero entries: + [2, 1] = 1.0 + [5, 1] = 1.0 + [3, 2] = 1.0 + [4, 2] = 1.0 + [2, 3] = 1.0 + [3, 5] = 1.0 + [4, 5] = 1.0 + [5, 5] = 1.0 +``` + +Note the difference from [`speye`](:func:`speye`). """ spones{T}(S::SparseMatrixCSC{T}) = SparseMatrixCSC(S.m, S.n, copy(S.colptr), copy(S.rowval), ones(T, S.colptr[end]-1)) @@ -1240,7 +1266,31 @@ speye(m::Integer, n::Integer) = speye(Float64, m, n) """ speye(S) -Create a sparse identity matrix with the same structure as that of `S`. +Create a sparse identity matrix with the same structure as that of `S`. + +```jldoctest +julia> A = sprand(5,6,0.2) +5×6 sparse matrix with 9 Float64 nonzero entries: + [1, 1] = 0.102874 + [2, 1] = 0.780098 + [1, 2] = 0.610378 + [1, 3] = 0.422308 + [3, 3] = 0.546398 + [4, 3] = 0.43053 + [5, 3] = 0.909283 + [2, 4] = 0.391321 + [5, 6] = 0.97785 + +julia> speye(A) +5×6 sparse matrix with 5 Float64 nonzero entries: + [1, 1] = 1.0 + [2, 2] = 1.0 + [3, 3] = 1.0 + [4, 4] = 1.0 + [5, 5] = 1.0 +``` + +Note the difference from [`spones`](:func:`spones`). """ speye{T}(S::SparseMatrixCSC{T}) = speye(T, size(S, 1), size(S, 2)) eye(S::SparseMatrixCSC) = speye(S) @@ -3461,6 +3511,19 @@ Construct a sparse diagonal matrix. `B` is a tuple of vectors containing the dia one diagonal, `B` can be a vector (instead of a tuple) and `d` can be the diagonal position (instead of a tuple), defaulting to 0 (diagonal). Optionally, `m` and `n` specify the size of the resulting sparse matrix. + +```jldoctest +julia> spdiagm((rand(4), rand(4)), (-1, 1)) +5×5 sparse matrix with 8 Float64 nonzero entries: + [2, 1] = 0.962245 + [1, 2] = 0.919341 + [3, 2] = 0.59239 + [2, 3] = 0.628924 + [4, 3] = 0.694011 + [3, 4] = 0.0660923 + [5, 4] = 0.494409 + [4, 5] = 0.54209 +``` """ function spdiagm(B, d, m::Integer, n::Integer) (I,J,V) = spdiagm_internal(B, d) diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index f45ff006269a5..91ed246b883c5 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -1198,41 +1198,125 @@ to/from the latter via ``Array(bitarray)`` and ``BitArray(array)``, respectively Performs a bitwise not operation on ``B``\ . See :ref:`~ operator <~>`\ . -.. function:: rol!(dest::BitArray{1}, src::BitArray{1}, i::Integer) -> BitArray{1} + .. doctest:: + + julia> A = bitrand(3,5) + 3×5 BitArray{2}: + true true true true true + false false false true false + true false false false true + + julia> flipbits!(A) + 3×5 BitArray{2}: + false false false false false + true true true false true + false true true true false + +.. function:: rol!(dest::BitVector, src::BitVector, i::Integer) -> BitVector .. Docstring generated from Julia source Performs a left rotation operation on ``src`` and put the result into ``dest``\ . -.. function:: rol!(B::BitArray{1}, i::Integer) -> BitArray{1} + .. doctest:: + + julia> a = bitrand(5) + 5-element BitArray{1}: + false + false + true + true + true + + julia> b = falses(5); + + julia> rol!(b,a,2) + 5-element BitArray{1}: + true + true + true + false + false + + julia> rol!(b,a,3) + 5-element BitArray{1}: + true + true + false + false + true + +.. function:: rol!(B::BitVector, i::Integer) -> BitVector .. Docstring generated from Julia source Performs a left rotation operation on ``B``\ . -.. function:: rol(B::BitArray{1}, i::Integer) -> BitArray{1} + ```jldoctest julia> a = bitrand(5) 5-element BitArray{1}: true false false true true + + julia> rol!(a,4) 5-element BitArray{1}: true true false false true + +.. function:: rol(B::BitVector, i::Integer) -> BitVector .. Docstring generated from Julia source - Performs a left rotation operation. + Performs a left rotation operation without modifying ``B``\ . See also :func:`rol!`\ . -.. function:: ror!(dest::BitArray{1}, src::BitArray{1}, i::Integer) -> BitArray{1} +.. function:: ror!(dest::BitVector, src::BitVector, i::Integer) -> BitVector .. Docstring generated from Julia source Performs a right rotation operation on ``src`` and put the result into ``dest``\ . -.. function:: ror!(B::BitArray{1}, i::Integer) -> BitArray{1} + .. doctest:: + + julia> a = bitrand(5) + 5-element BitArray{1}: + false + false + true + true + true + + julia> b = falses(5); + + julia> ror!(b,a,2) + 5-element BitArray{1}: + true + true + false + false + true + +.. function:: ror!(B::BitVector, i::Integer) -> BitVector .. Docstring generated from Julia source Performs a right rotation operation on ``B``\ . -.. function:: ror(B::BitArray{1}, i::Integer) -> BitArray{1} + .. doctest:: + + julia> a = bitrand(5) + 5-element BitArray{1}: + false + true + false + true + false + + julia> ror!(a,3) + 5-element BitArray{1}: + false + true + false + false + true + +.. function:: ror(B::BitVector, i::Integer) -> BitVector .. Docstring generated from Julia source - Performs a right rotation operation. + Performs a right rotation operation without modifying ``B``\ . See also :func:`ror!`\ . .. _stdlib-sparse: @@ -1246,7 +1330,7 @@ dense counterparts. The following functions are specific to sparse arrays. .. Docstring generated from Julia source - Create a sparse matrix ``S`` of dimensions ``m x n`` such that ``S[I[k], J[k]] = V[k]``\ . The ``combine`` function is used to combine duplicates. If ``m`` and ``n`` are not specified, they are set to ``maximum(I)`` and ``maximum(J)`` respectively. If the ``combine`` function is not supplied, ``combine`` defaults to ``+`` unless the elements of ``V`` are Booleans in which case ``combine`` defaults to ``|``\ . All elements of ``I`` must satisfy ``1 <= I[k] <= m``\ , and all elements of ``J`` must satisfy ``1 <= J[k] <= n``\ . Numerical zeros in (``I``\ , ``J``\ , ``V``\ ) are retained as structural nonzeros; to drop numerical zeros, use ``dropzeros!``\ . + Create a sparse matrix ``S`` of dimensions ``m x n`` such that ``S[I[k], J[k]] = V[k]``\ . The ``combine`` function is used to combine duplicates. If ``m`` and ``n`` are not specified, they are set to ``maximum(I)`` and ``maximum(J)`` respectively. If the ``combine`` function is not supplied, ``combine`` defaults to ``+`` unless the elements of ``V`` are Booleans in which case ``combine`` defaults to ``|``\ . All elements of ``I`` must satisfy ``1 <= I[k] <= m``\ , and all elements of ``J`` must satisfy ``1 <= J[k] <= n``\ . Numerical zeros in (``I``\ , ``J``\ , ``V``\ ) are retained as structural nonzeros; to drop numerical zeros, use :func:`dropzeros!`\ . For additional documentation and an expert driver, see ``Base.SparseArrays.sparse!``\ . @@ -1304,6 +1388,32 @@ dense counterparts. The following functions are specific to sparse arrays. Create a sparse array with the same structure as that of ``S``\ , but with every nonzero element having the value ``1.0``\ . + .. doctest:: + + julia> A = sprand(5,6,0.2) + 5×6 sparse matrix with 8 Float64 nonzero entries: + [2, 1] = 0.639431 + [5, 1] = 0.881209 + [3, 2] = 0.355834 + [4, 2] = 0.904768 + [2, 3] = 0.760943 + [3, 5] = 0.525942 + [4, 5] = 0.936283 + [5, 5] = 0.432364 + + julia> spones(A) + 5×6 sparse matrix with 8 Float64 nonzero entries: + [2, 1] = 1.0 + [5, 1] = 1.0 + [3, 2] = 1.0 + [4, 2] = 1.0 + [2, 3] = 1.0 + [3, 5] = 1.0 + [4, 5] = 1.0 + [5, 5] = 1.0 + + Note the difference from :func:`speye`\ . + .. function:: speye([type,]m[,n]) .. Docstring generated from Julia source @@ -1314,7 +1424,31 @@ dense counterparts. The following functions are specific to sparse arrays. .. Docstring generated from Julia source - Create a sparse identity matrix with the same structure as that of ``S``\ . + Create a sparse identity matrix with the same structure as that of ``S``\ . + + .. doctest:: + + julia> A = sprand(5,6,0.2) + 5×6 sparse matrix with 9 Float64 nonzero entries: + [1, 1] = 0.102874 + [2, 1] = 0.780098 + [1, 2] = 0.610378 + [1, 3] = 0.422308 + [3, 3] = 0.546398 + [4, 3] = 0.43053 + [5, 3] = 0.909283 + [2, 4] = 0.391321 + [5, 6] = 0.97785 + + julia> speye(A) + 5×6 sparse matrix with 5 Float64 nonzero entries: + [1, 1] = 1.0 + [2, 2] = 1.0 + [3, 3] = 1.0 + [4, 4] = 1.0 + [5, 5] = 1.0 + + Note the difference from :func:`spones`\ . .. function:: spdiagm(B, d[, m, n]) @@ -1322,6 +1456,19 @@ dense counterparts. The following functions are specific to sparse arrays. Construct a sparse diagonal matrix. ``B`` is a tuple of vectors containing the diagonals and ``d`` is a tuple containing the positions of the diagonals. In the case the input contains only one diagonal, ``B`` can be a vector (instead of a tuple) and ``d`` can be the diagonal position (instead of a tuple), defaulting to 0 (diagonal). Optionally, ``m`` and ``n`` specify the size of the resulting sparse matrix. + .. doctest:: + + julia> spdiagm((rand(4), rand(4)), (-1, 1)) + 5×5 sparse matrix with 8 Float64 nonzero entries: + [2, 1] = 0.962245 + [1, 2] = 0.919341 + [3, 2] = 0.59239 + [2, 3] = 0.628924 + [4, 3] = 0.694011 + [3, 4] = 0.0660923 + [5, 4] = 0.494409 + [4, 5] = 0.54209 + .. function:: sprand([rng],[type],m,[n],p::AbstractFloat,[rfn]) .. Docstring generated from Julia source @@ -1338,19 +1485,19 @@ dense counterparts. The following functions are specific to sparse arrays. .. Docstring generated from Julia source - Return a vector of the structural nonzero values in sparse array ``A``\ . This includes zeros that are explicitly stored in the sparse array. The returned vector points directly to the internal nonzero storage of ``A``\ , and any modifications to the returned vector will mutate ``A`` as well. See ``rowvals(A)`` and ``nzrange(A, col)``\ . + Return a vector of the structural nonzero values in sparse array ``A``\ . This includes zeros that are explicitly stored in the sparse array. The returned vector points directly to the internal nonzero storage of ``A``\ , and any modifications to the returned vector will mutate ``A`` as well. See :func:`rowvals` and :func:`nzrange`\ . .. function:: rowvals(A::SparseMatrixCSC) .. Docstring generated from Julia source - Return a vector of the row indices of ``A``\ . Any modifications to the returned vector will mutate ``A`` as well. Providing access to how the row indices are stored internally can be useful in conjunction with iterating over structural nonzero values. See also ``nonzeros(A)`` and ``nzrange(A, col)``\ . + Return a vector of the row indices of ``A``\ . Any modifications to the returned vector will mutate ``A`` as well. Providing access to how the row indices are stored internally can be useful in conjunction with iterating over structural nonzero values. See also :func:`nonzeros` and :func:`nzrange`\ . .. function:: nzrange(A::SparseMatrixCSC, col) .. Docstring generated from Julia source - Return the range of indices to the structural nonzero values of a sparse matrix column. In conjunction with ``nonzeros(A)`` and ``rowvals(A)``\ , this allows for convenient iterating over a sparse matrix : + Return the range of indices to the structural nonzero values of a sparse matrix column. In conjunction with :func:`nonzeros` and :func:`rowvals`\ , this allows for convenient iterating over a sparse matrix : .. code-block:: julia @@ -1372,7 +1519,7 @@ dense counterparts. The following functions are specific to sparse arrays. Removes stored numerical zeros from ``A``\ , optionally trimming resulting excess space from ``A.rowval`` and ``A.nzval`` when ``trim`` is ``true``\ . - For an out-of-place version, see :func:`Base.SparseArrays.dropzeros`\ . For algorithmic information, see :func:`Base.SparseArrays.fkeep!`\ . + For an out-of-place version, see :func:`dropzeros`\ . For algorithmic information, see :func:`Base.SparseArrays.fkeep!`\ . .. function:: dropzeros(A::SparseMatrixCSC, trim::Bool = true) @@ -1380,7 +1527,7 @@ dense counterparts. The following functions are specific to sparse arrays. Generates a copy of ``A`` and removes stored numerical zeros from that copy, optionally trimming excess space from the result's ``rowval`` and ``nzval`` arrays when ``trim`` is ``true``\ . - For an in-place version and algorithmic information, see :func:`Base.SparseArrays.dropzeros!`\ . + For an in-place version and algorithmic information, see :func:`dropzeros!`\ . .. function:: dropzeros!(x::SparseVector, trim::Bool = true) From 775c34e36c9392f1947d9ced0ae413709f3e53b6 Mon Sep 17 00:00:00 2001 From: kshyatt Date: Thu, 4 Aug 2016 09:27:23 -0700 Subject: [PATCH 20/80] Added examples for ones, eye. Fixed up BitArray docs. Removed calls to `rand` in the doctests. (cherry picked from commit 6df3ec0a4410e9f30e0bcc15595722cc8f00e16d) ref #17797 --- base/array.jl | 63 ++++++++++++ base/bitarray.jl | 135 ++++++++++++------------ base/docs/helpdb/Base.jl | 35 ------- base/sparse/sparsematrix.jl | 72 ++++++------- doc/stdlib/arrays.rst | 198 +++++++++++++++++++----------------- 5 files changed, 262 insertions(+), 241 deletions(-) diff --git a/base/array.jl b/base/array.jl index 783b93e0e5956..3a4ac020af589 100644 --- a/base/array.jl +++ b/base/array.jl @@ -171,7 +171,42 @@ for (fname, felt) in ((:zeros,:zero), (:ones,:one)) ($fname){T}(A::AbstractArray{T}) = fill!(similar(A), ($felt)(T)) end end +""" + ones([T::Type=Float64,] dims) + +Create an array of all ones of specified type. The type defaults to `Float64` if not specified. +""" +function ones(dims::Dims) end + +""" + ones(A) + +Create an array of all ones with the same element type and shape as `A`. + +```jldoctest +julia> A = [1 2 3; 4 5 6; 7 8 9] +3×3 Array{Int64,2}: + 1 2 3 + 4 5 6 + 7 8 9 + +julia> ones(A) +3×3 Array{Int64,2}: + 1 1 1 + 1 1 1 + 1 1 1 +``` + +Note the difference from [`eye`](:func:`eye`). +""" +function ones(A) end + +""" + eye([T::Type=Float64,] m::Integer, n::Integer) +`m`-by-`n` identity matrix. +The default element type is `Float64`. +""" function eye(T::Type, m::Integer, n::Integer) a = zeros(T,m,n) for i = 1:min(m,n) @@ -181,7 +216,35 @@ function eye(T::Type, m::Integer, n::Integer) end eye(m::Integer, n::Integer) = eye(Float64, m, n) eye(T::Type, n::Integer) = eye(T, n, n) +""" + eye([T::Type=Float64,] n::Integer) + +`n`-by-`n` identity matrix. +The default element type is `Float64`. +""" eye(n::Integer) = eye(Float64, n) + +""" + eye(A) + +Constructs an identity matrix of the same dimensions and type as `A`. + +```jldoctest +julia> A = [1 2 3; 4 5 6; 7 8 9] +3×3 Array{Int64,2}: + 1 2 3 + 4 5 6 + 7 8 9 + +julia> eye(A) +3×3 Array{Int64,2}: + 1 0 0 + 0 1 0 + 0 0 1 +``` + +Note the difference from [`ones`](:func:`ones`). +""" eye{T}(x::AbstractMatrix{T}) = eye(T, size(x, 1), size(x, 2)) function one{T}(x::AbstractMatrix{T}) diff --git a/base/bitarray.jl b/base/bitarray.jl index 95115cb3501d9..3ed1fa225b0cc 100644 --- a/base/bitarray.jl +++ b/base/bitarray.jl @@ -1427,35 +1427,8 @@ end """ rol!(dest::BitVector, src::BitVector, i::Integer) -> BitVector -Performs a left rotation operation on `src` and put the result into `dest`. - -```jldoctest -julia> a = bitrand(5) -5-element BitArray{1}: - false - false - true - true - true - -julia> b = falses(5); - -julia> rol!(b,a,2) -5-element BitArray{1}: - true - true - true - false - false - -julia> rol!(b,a,3) -5-element BitArray{1}: - true - true - false - false - true -``` +Performs a left rotation operation on `src` and puts the result into `dest`. +`i` controls how far to rotate each bit. """ function rol!(dest::BitVector, src::BitVector, i::Integer) length(dest) == length(src) || throw(ArgumentError("destination and source should be of same size")) @@ -1472,46 +1445,38 @@ end """ rol!(B::BitVector, i::Integer) -> BitVector -Performs a left rotation operation on `B`. +Performs a left rotation operation in-place on `B`. +`i` controls how far to rotate each bit. +""" +function rol!(B::BitVector, i::Integer) + return rol!(B, B, i) +end + +""" + rol(B::BitVector, i::Integer) -> BitVector + +Performs a left rotation operation, returning a new `BitVector`. +`i` controls how far to rotate each bit. +See also [`rol!`](:func:`rol!`). ```jldoctest -julia> a = bitrand(5) +julia> A = BitArray([true, true, false, false, true]) 5-element BitArray{1}: true + true false false true - true -julia> rol!(a,4) +julia> rol(A,1) 5-element BitArray{1}: true - true false false true -""" -function rol!(B::BitVector, i::Integer) - return rol!(B, B, i) -end - -""" - rol(B::BitVector, i::Integer) -> BitVector - -Performs a left rotation operation without modifying `B`. -See also [`rol!`](:func:`rol!`). -""" -function rol(B::BitVector, i::Integer) - return rol!(similar(B), B, i) -end - -""" - ror!(dest::BitVector, src::BitVector, i::Integer) -> BitVector - -Performs a right rotation operation on `src` and put the result into `dest`. + true -```jldoctest -julia> a = bitrand(5) +julia> rol(A,2) 5-element BitArray{1}: false false @@ -1519,9 +1484,7 @@ julia> a = bitrand(5) true true -julia> b = falses(5); - -julia> ror!(b,a,2) +julia> rol(A,5) 5-element BitArray{1}: true true @@ -1530,6 +1493,16 @@ julia> ror!(b,a,2) true ``` """ +function rol(B::BitVector, i::Integer) + return rol!(similar(B), B, i) +end + +""" + ror!(dest::BitVector, src::BitVector, i::Integer) -> BitVector + +Performs a right rotation operation on `src` and puts the result into `dest`. +`i` controls how far to rotate each bit. +""" function ror!(dest::BitVector, src::BitVector, i::Integer) length(dest) == length(src) || throw(ArgumentError("destination and source should be of same size")) n = length(dest) @@ -1545,36 +1518,54 @@ end """ ror!(B::BitVector, i::Integer) -> BitVector -Performs a right rotation operation on `B`. +Performs a right rotation operation in-place on `B`. +`i` controls how far to rotate each bit. +""" +function ror!(B::BitVector, i::Integer) + return ror!(B, B, i) +end + +""" + ror(B::BitVector, i::Integer) -> BitVector + +Performs a right rotation operation on `B`, returning a new `BitVector`. +`i` controls how far to rotate each bit. +See also [`ror!`](:func:`ror!`). ```jldoctest -julia> a = bitrand(5) +julia> A = BitArray([true, true, false, false, true]) 5-element BitArray{1}: - false true + true + false false true + +julia> ror(A,1) +5-element BitArray{1}: + true + true + true + false false -julia> ror!(a,3) +julia> ror(A,2) 5-element BitArray{1}: false true + true + true + false + +julia> ror(A,5) +5-element BitArray{1}: + true + true false false true ``` """ -function ror!(B::BitVector, i::Integer) - return ror!(B, B, i) -end - -""" - ror(B::BitVector, i::Integer) -> BitVector - -Performs a right rotation operation without modifying `B`. -See also [`ror!`](:func:`ror!`). -""" function ror(B::BitVector, i::Integer) return ror!(similar(B), B, i) end diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 51855d833c1db..86aef17f872f7 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -1195,20 +1195,6 @@ of the problem that is solved on each processor. """ peakflops -""" - ones(type, dims) - -Create an array of all ones of specified type. The type defaults to `Float64` if not specified. -""" -ones(t,dims) - -""" - ones(A) - -Create an array of all ones with the same element type and shape as `A`. -""" -ones(A) - """ ind2chr(string, i) @@ -4003,27 +3989,6 @@ Euler integral of the first kind ``\\operatorname{B}(x,y) = \\Gamma(x)\\Gamma(y) """ beta -""" - eye(n) - -`n`-by-`n` identity matrix. -""" -eye(n::Int) - -""" - eye(m, n) - -`m`-by-`n` identity matrix. -""" -eye(m, n) - -""" - eye(A) - -Constructs an identity matrix of the same dimensions and type as `A`. -""" -eye(A) - """ diagind(M[, k]) diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index fa25fa2e64ac4..b943dc5dee93d 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -1214,27 +1214,19 @@ Create a sparse array with the same structure as that of `S`, but with every non element having the value `1.0`. ```jldoctest -julia> A = sprand(5,6,0.2) -5×6 sparse matrix with 8 Float64 nonzero entries: - [2, 1] = 0.639431 - [5, 1] = 0.881209 - [3, 2] = 0.355834 - [4, 2] = 0.904768 - [2, 3] = 0.760943 - [3, 5] = 0.525942 - [4, 5] = 0.936283 - [5, 5] = 0.432364 +julia> A = sparse([1,2,3,4],[2,4,3,1],[5.,4.,3.,2.]) +4×4 sparse matrix with 4 Float64 nonzero entries: + [4, 1] = 2.0 + [1, 2] = 5.0 + [3, 3] = 3.0 + [2, 4] = 4.0 julia> spones(A) -5×6 sparse matrix with 8 Float64 nonzero entries: - [2, 1] = 1.0 - [5, 1] = 1.0 - [3, 2] = 1.0 - [4, 2] = 1.0 - [2, 3] = 1.0 - [3, 5] = 1.0 - [4, 5] = 1.0 - [5, 5] = 1.0 +4×4 sparse matrix with 4 Float64 nonzero entries: + [4, 1] = 1.0 + [1, 2] = 1.0 + [3, 3] = 1.0 + [2, 4] = 1.0 ``` Note the difference from [`speye`](:func:`speye`). @@ -1266,28 +1258,22 @@ speye(m::Integer, n::Integer) = speye(Float64, m, n) """ speye(S) -Create a sparse identity matrix with the same structure as that of `S`. +Create a sparse identity matrix with the same size as that of `S`. ```jldoctest -julia> A = sprand(5,6,0.2) -5×6 sparse matrix with 9 Float64 nonzero entries: - [1, 1] = 0.102874 - [2, 1] = 0.780098 - [1, 2] = 0.610378 - [1, 3] = 0.422308 - [3, 3] = 0.546398 - [4, 3] = 0.43053 - [5, 3] = 0.909283 - [2, 4] = 0.391321 - [5, 6] = 0.97785 +julia> A = sparse([1,2,3,4],[2,4,3,1],[5.,4.,3.,2.]) +4×4 sparse matrix with 4 Float64 nonzero entries: + [4, 1] = 2.0 + [1, 2] = 5.0 + [3, 3] = 3.0 + [2, 4] = 4.0 julia> speye(A) -5×6 sparse matrix with 5 Float64 nonzero entries: +4×4 sparse matrix with 4 Float64 nonzero entries: [1, 1] = 1.0 [2, 2] = 1.0 [3, 3] = 1.0 [4, 4] = 1.0 - [5, 5] = 1.0 ``` Note the difference from [`spones`](:func:`spones`). @@ -3513,16 +3499,16 @@ one diagonal, `B` can be a vector (instead of a tuple) and `d` can be the diagon of the resulting sparse matrix. ```jldoctest -julia> spdiagm((rand(4), rand(4)), (-1, 1)) -5×5 sparse matrix with 8 Float64 nonzero entries: - [2, 1] = 0.962245 - [1, 2] = 0.919341 - [3, 2] = 0.59239 - [2, 3] = 0.628924 - [4, 3] = 0.694011 - [3, 4] = 0.0660923 - [5, 4] = 0.494409 - [4, 5] = 0.54209 +julia> spdiagm(([1,2,3,4],[4,3,2,1]),(-1,1)) +5×5 sparse matrix with 8 Int64 nonzero entries: + [2, 1] = 1 + [1, 2] = 4 + [3, 2] = 2 + [2, 3] = 3 + [4, 3] = 3 + [3, 4] = 2 + [5, 4] = 4 + [4, 5] = 1 ``` """ function spdiagm(B, d, m::Integer, n::Integer) diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index 91ed246b883c5..b51ab8bae7118 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -187,7 +187,7 @@ Constructors Create an array of all zeros with the same element type and shape as ``A``\ . -.. function:: ones(type, dims) +.. function:: ones([T::Type=Float64,] dims) .. Docstring generated from Julia source @@ -199,6 +199,22 @@ Constructors Create an array of all ones with the same element type and shape as ``A``\ . + .. doctest:: + + julia> A = [1 2 3; 4 5 6; 7 8 9] + 3×3 Array{Int64,2}: + 1 2 3 + 4 5 6 + 7 8 9 + + julia> ones(A) + 3×3 Array{Int64,2}: + 1 1 1 + 1 1 1 + 1 1 1 + + Note the difference from :func:`eye`\ . + .. function:: trues(dims) .. Docstring generated from Julia source @@ -309,17 +325,17 @@ Constructors Change the type-interpretation of a block of memory. For example, ``reinterpret(Float32, UInt32(7))`` interprets the 4 bytes corresponding to ``UInt32(7)`` as a ``Float32``\ . For arrays, this constructs an array with the same binary data as the given array, but with the specified element type. -.. function:: eye(n) +.. function:: eye([T::Type=Float64,] n::Integer) .. Docstring generated from Julia source - ``n``\ -by-``n`` identity matrix. + ``n``\ -by-``n`` identity matrix. The default element type is ``Float64``\ . -.. function:: eye(m, n) +.. function:: eye([T::Type=Float64,] m::Integer, n::Integer) .. Docstring generated from Julia source - ``m``\ -by-``n`` identity matrix. + ``m``\ -by-``n`` identity matrix. The default element type is ``Float64``\ . .. function:: eye(A) @@ -327,6 +343,22 @@ Constructors Constructs an identity matrix of the same dimensions and type as ``A``\ . + .. doctest:: + + julia> A = [1 2 3; 4 5 6; 7 8 9] + 3×3 Array{Int64,2}: + 1 2 3 + 4 5 6 + 7 8 9 + + julia> eye(A) + 3×3 Array{Int64,2}: + 1 0 0 + 0 1 0 + 0 0 1 + + Note the difference from :func:`ones`\ . + .. function:: linspace(start, stop, n=50) .. Docstring generated from Julia source @@ -1216,29 +1248,47 @@ to/from the latter via ``Array(bitarray)`` and ``BitArray(array)``, respectively .. Docstring generated from Julia source - Performs a left rotation operation on ``src`` and put the result into ``dest``\ . + Performs a left rotation operation on ``src`` and puts the result into ``dest``\ . ``i`` controls how far to rotate each bit. + +.. function:: rol!(B::BitVector, i::Integer) -> BitVector + + .. Docstring generated from Julia source + + Performs a left rotation operation in-place on ``B``\ . ``i`` controls how far to rotate each bit. + +.. function:: rol(B::BitVector, i::Integer) -> BitVector + + .. Docstring generated from Julia source + + Performs a left rotation operation, returning a new ``BitVector``\ . ``i`` controls how far to rotate each bit. See also :func:`rol!`\ . .. doctest:: - julia> a = bitrand(5) + julia> A = BitArray([true, true, false, false, true]) 5-element BitArray{1}: - false - false true true + false + false true - julia> b = falses(5); - - julia> rol!(b,a,2) + julia> rol(A,1) 5-element BitArray{1}: true + false + false true true + + julia> rol(A,2) + 5-element BitArray{1}: false false + true + true + true - julia> rol!(b,a,3) + julia> rol(A,5) 5-element BitArray{1}: true true @@ -1246,78 +1296,58 @@ to/from the latter via ``Array(bitarray)`` and ``BitArray(array)``, respectively false true -.. function:: rol!(B::BitVector, i::Integer) -> BitVector +.. function:: ror!(dest::BitVector, src::BitVector, i::Integer) -> BitVector .. Docstring generated from Julia source - Performs a left rotation operation on ``B``\ . - - ```jldoctest julia> a = bitrand(5) 5-element BitArray{1}: true false false true true + Performs a right rotation operation on ``src`` and puts the result into ``dest``\ . ``i`` controls how far to rotate each bit. - julia> rol!(a,4) 5-element BitArray{1}: true true false false true - -.. function:: rol(B::BitVector, i::Integer) -> BitVector +.. function:: ror!(B::BitVector, i::Integer) -> BitVector .. Docstring generated from Julia source - Performs a left rotation operation without modifying ``B``\ . See also :func:`rol!`\ . + Performs a right rotation operation in-place on ``B``\ . ``i`` controls how far to rotate each bit. -.. function:: ror!(dest::BitVector, src::BitVector, i::Integer) -> BitVector +.. function:: ror(B::BitVector, i::Integer) -> BitVector .. Docstring generated from Julia source - Performs a right rotation operation on ``src`` and put the result into ``dest``\ . + Performs a right rotation operation on ``B``\ , returning a new ``BitVector``\ . ``i`` controls how far to rotate each bit. See also :func:`ror!`\ . .. doctest:: - julia> a = bitrand(5) + julia> A = BitArray([true, true, false, false, true]) 5-element BitArray{1}: - false - false true true + false + false true - julia> b = falses(5); - - julia> ror!(b,a,2) + julia> ror(A,1) 5-element BitArray{1}: true true + true false false - true -.. function:: ror!(B::BitVector, i::Integer) -> BitVector - - .. Docstring generated from Julia source - - Performs a right rotation operation on ``B``\ . - - .. doctest:: - - julia> a = bitrand(5) + julia> ror(A,2) 5-element BitArray{1}: false true - false + true true false - julia> ror!(a,3) + julia> ror(A,5) 5-element BitArray{1}: - false + true true false false true -.. function:: ror(B::BitVector, i::Integer) -> BitVector - - .. Docstring generated from Julia source - - Performs a right rotation operation without modifying ``B``\ . See also :func:`ror!`\ . - .. _stdlib-sparse: Sparse Vectors and Matrices @@ -1390,27 +1420,19 @@ dense counterparts. The following functions are specific to sparse arrays. .. doctest:: - julia> A = sprand(5,6,0.2) - 5×6 sparse matrix with 8 Float64 nonzero entries: - [2, 1] = 0.639431 - [5, 1] = 0.881209 - [3, 2] = 0.355834 - [4, 2] = 0.904768 - [2, 3] = 0.760943 - [3, 5] = 0.525942 - [4, 5] = 0.936283 - [5, 5] = 0.432364 + julia> A = sparse([1,2,3,4],[2,4,3,1],[5.,4.,3.,2.]) + 4×4 sparse matrix with 4 Float64 nonzero entries: + [4, 1] = 2.0 + [1, 2] = 5.0 + [3, 3] = 3.0 + [2, 4] = 4.0 julia> spones(A) - 5×6 sparse matrix with 8 Float64 nonzero entries: - [2, 1] = 1.0 - [5, 1] = 1.0 - [3, 2] = 1.0 - [4, 2] = 1.0 - [2, 3] = 1.0 - [3, 5] = 1.0 - [4, 5] = 1.0 - [5, 5] = 1.0 + 4×4 sparse matrix with 4 Float64 nonzero entries: + [4, 1] = 1.0 + [1, 2] = 1.0 + [3, 3] = 1.0 + [2, 4] = 1.0 Note the difference from :func:`speye`\ . @@ -1424,29 +1446,23 @@ dense counterparts. The following functions are specific to sparse arrays. .. Docstring generated from Julia source - Create a sparse identity matrix with the same structure as that of ``S``\ . + Create a sparse identity matrix with the same size as that of ``S``\ . .. doctest:: - julia> A = sprand(5,6,0.2) - 5×6 sparse matrix with 9 Float64 nonzero entries: - [1, 1] = 0.102874 - [2, 1] = 0.780098 - [1, 2] = 0.610378 - [1, 3] = 0.422308 - [3, 3] = 0.546398 - [4, 3] = 0.43053 - [5, 3] = 0.909283 - [2, 4] = 0.391321 - [5, 6] = 0.97785 + julia> A = sparse([1,2,3,4],[2,4,3,1],[5.,4.,3.,2.]) + 4×4 sparse matrix with 4 Float64 nonzero entries: + [4, 1] = 2.0 + [1, 2] = 5.0 + [3, 3] = 3.0 + [2, 4] = 4.0 julia> speye(A) - 5×6 sparse matrix with 5 Float64 nonzero entries: + 4×4 sparse matrix with 4 Float64 nonzero entries: [1, 1] = 1.0 [2, 2] = 1.0 [3, 3] = 1.0 [4, 4] = 1.0 - [5, 5] = 1.0 Note the difference from :func:`spones`\ . @@ -1458,16 +1474,16 @@ dense counterparts. The following functions are specific to sparse arrays. .. doctest:: - julia> spdiagm((rand(4), rand(4)), (-1, 1)) - 5×5 sparse matrix with 8 Float64 nonzero entries: - [2, 1] = 0.962245 - [1, 2] = 0.919341 - [3, 2] = 0.59239 - [2, 3] = 0.628924 - [4, 3] = 0.694011 - [3, 4] = 0.0660923 - [5, 4] = 0.494409 - [4, 5] = 0.54209 + julia> spdiagm(([1,2,3,4],[4,3,2,1]),(-1,1)) + 5×5 sparse matrix with 8 Int64 nonzero entries: + [2, 1] = 1 + [1, 2] = 4 + [3, 2] = 2 + [2, 3] = 3 + [4, 3] = 3 + [3, 4] = 2 + [5, 4] = 4 + [4, 5] = 1 .. function:: sprand([rng],[type],m,[n],p::AbstractFloat,[rfn]) From 6903268ddb51b85ec737bc0e89b3df0607784e8d Mon Sep 17 00:00:00 2001 From: kshyatt Date: Thu, 4 Aug 2016 16:44:34 -0700 Subject: [PATCH 21/80] Doctests passed, gave up on ones (cherry picked from commit 071cd66aebf5968b9851071c70495ecaa4219a37) ref #17797 --- base/array.jl | 29 ------------- base/bitarray.jl | 16 ++++---- base/docs/helpdb/Base.jl | 28 +++++++++++++ base/sparse/sparsematrix.jl | 48 +++++++++++----------- doc/stdlib/arrays.rst | 82 +++++++++++++++---------------------- 5 files changed, 91 insertions(+), 112 deletions(-) diff --git a/base/array.jl b/base/array.jl index 3a4ac020af589..8c753b3e3924d 100644 --- a/base/array.jl +++ b/base/array.jl @@ -171,35 +171,6 @@ for (fname, felt) in ((:zeros,:zero), (:ones,:one)) ($fname){T}(A::AbstractArray{T}) = fill!(similar(A), ($felt)(T)) end end -""" - ones([T::Type=Float64,] dims) - -Create an array of all ones of specified type. The type defaults to `Float64` if not specified. -""" -function ones(dims::Dims) end - -""" - ones(A) - -Create an array of all ones with the same element type and shape as `A`. - -```jldoctest -julia> A = [1 2 3; 4 5 6; 7 8 9] -3×3 Array{Int64,2}: - 1 2 3 - 4 5 6 - 7 8 9 - -julia> ones(A) -3×3 Array{Int64,2}: - 1 1 1 - 1 1 1 - 1 1 1 -``` - -Note the difference from [`eye`](:func:`eye`). -""" -function ones(A) end """ eye([T::Type=Float64,] m::Integer, n::Integer) diff --git a/base/bitarray.jl b/base/bitarray.jl index 3ed1fa225b0cc..a059248f4ccc5 100644 --- a/base/bitarray.jl +++ b/base/bitarray.jl @@ -1011,17 +1011,15 @@ end Performs a bitwise not operation on `B`. See [`~`](:ref:`~ operator <~>`). ```jldoctest -julia> A = bitrand(3,5) -3×5 BitArray{2}: - true true true true true - false false false true false - true false false false true +julia> A = trues(2,2) +2×2 BitArray{2}: + true true + true true julia> flipbits!(A) -3×5 BitArray{2}: - false false false false false - true true true false true - false true true true false +2×2 BitArray{2}: + false false + false false ``` """ function flipbits!(B::BitArray) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 86aef17f872f7..4e6fe7e4fcfd0 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -1195,6 +1195,20 @@ of the problem that is solved on each processor. """ peakflops +""" + ones(type, dims) + +Create an array of all ones of specified type. The type defaults to `Float64` if not specified. +""" +ones(t,dims) + +""" + ones(A) + +Create an array of all ones with the same element type and shape as `A`. +""" +ones(A) + """ ind2chr(string, i) @@ -3989,6 +4003,20 @@ Euler integral of the first kind ``\\operatorname{B}(x,y) = \\Gamma(x)\\Gamma(y) """ beta +""" + eye(n) + +`n`-by-`n` identity matrix. +""" +eye(n::Int) + +""" + eye(m, n) + +`m`-by-`n` identity matrix. +""" +eye(m, n) + """ diagind(M[, k]) diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index b943dc5dee93d..0e3f2bf68c174 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -1216,17 +1216,17 @@ element having the value `1.0`. ```jldoctest julia> A = sparse([1,2,3,4],[2,4,3,1],[5.,4.,3.,2.]) 4×4 sparse matrix with 4 Float64 nonzero entries: - [4, 1] = 2.0 - [1, 2] = 5.0 - [3, 3] = 3.0 - [2, 4] = 4.0 + [4, 1] = 2.0 + [1, 2] = 5.0 + [3, 3] = 3.0 + [2, 4] = 4.0 julia> spones(A) 4×4 sparse matrix with 4 Float64 nonzero entries: - [4, 1] = 1.0 - [1, 2] = 1.0 - [3, 3] = 1.0 - [2, 4] = 1.0 + [4, 1] = 1.0 + [1, 2] = 1.0 + [3, 3] = 1.0 + [2, 4] = 1.0 ``` Note the difference from [`speye`](:func:`speye`). @@ -1263,17 +1263,17 @@ Create a sparse identity matrix with the same size as that of `S`. ```jldoctest julia> A = sparse([1,2,3,4],[2,4,3,1],[5.,4.,3.,2.]) 4×4 sparse matrix with 4 Float64 nonzero entries: - [4, 1] = 2.0 - [1, 2] = 5.0 - [3, 3] = 3.0 - [2, 4] = 4.0 + [4, 1] = 2.0 + [1, 2] = 5.0 + [3, 3] = 3.0 + [2, 4] = 4.0 julia> speye(A) 4×4 sparse matrix with 4 Float64 nonzero entries: - [1, 1] = 1.0 - [2, 2] = 1.0 - [3, 3] = 1.0 - [4, 4] = 1.0 + [1, 1] = 1.0 + [2, 2] = 1.0 + [3, 3] = 1.0 + [4, 4] = 1.0 ``` Note the difference from [`spones`](:func:`spones`). @@ -3501,14 +3501,14 @@ of the resulting sparse matrix. ```jldoctest julia> spdiagm(([1,2,3,4],[4,3,2,1]),(-1,1)) 5×5 sparse matrix with 8 Int64 nonzero entries: - [2, 1] = 1 - [1, 2] = 4 - [3, 2] = 2 - [2, 3] = 3 - [4, 3] = 3 - [3, 4] = 2 - [5, 4] = 4 - [4, 5] = 1 + [2, 1] = 1 + [1, 2] = 4 + [3, 2] = 2 + [2, 3] = 3 + [4, 3] = 3 + [3, 4] = 2 + [5, 4] = 4 + [4, 5] = 1 ``` """ function spdiagm(B, d, m::Integer, n::Integer) diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index b51ab8bae7118..78cdb2e7e44ee 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -187,7 +187,7 @@ Constructors Create an array of all zeros with the same element type and shape as ``A``\ . -.. function:: ones([T::Type=Float64,] dims) +.. function:: ones(type, dims) .. Docstring generated from Julia source @@ -199,22 +199,6 @@ Constructors Create an array of all ones with the same element type and shape as ``A``\ . - .. doctest:: - - julia> A = [1 2 3; 4 5 6; 7 8 9] - 3×3 Array{Int64,2}: - 1 2 3 - 4 5 6 - 7 8 9 - - julia> ones(A) - 3×3 Array{Int64,2}: - 1 1 1 - 1 1 1 - 1 1 1 - - Note the difference from :func:`eye`\ . - .. function:: trues(dims) .. Docstring generated from Julia source @@ -1232,17 +1216,15 @@ to/from the latter via ``Array(bitarray)`` and ``BitArray(array)``, respectively .. doctest:: - julia> A = bitrand(3,5) - 3×5 BitArray{2}: - true true true true true - false false false true false - true false false false true + julia> A = trues(2,2) + 2×2 BitArray{2}: + true true + true true julia> flipbits!(A) - 3×5 BitArray{2}: - false false false false false - true true true false true - false true true true false + 2×2 BitArray{2}: + false false + false false .. function:: rol!(dest::BitVector, src::BitVector, i::Integer) -> BitVector @@ -1422,17 +1404,17 @@ dense counterparts. The following functions are specific to sparse arrays. julia> A = sparse([1,2,3,4],[2,4,3,1],[5.,4.,3.,2.]) 4×4 sparse matrix with 4 Float64 nonzero entries: - [4, 1] = 2.0 - [1, 2] = 5.0 - [3, 3] = 3.0 - [2, 4] = 4.0 + [4, 1] = 2.0 + [1, 2] = 5.0 + [3, 3] = 3.0 + [2, 4] = 4.0 julia> spones(A) 4×4 sparse matrix with 4 Float64 nonzero entries: - [4, 1] = 1.0 - [1, 2] = 1.0 - [3, 3] = 1.0 - [2, 4] = 1.0 + [4, 1] = 1.0 + [1, 2] = 1.0 + [3, 3] = 1.0 + [2, 4] = 1.0 Note the difference from :func:`speye`\ . @@ -1452,17 +1434,17 @@ dense counterparts. The following functions are specific to sparse arrays. julia> A = sparse([1,2,3,4],[2,4,3,1],[5.,4.,3.,2.]) 4×4 sparse matrix with 4 Float64 nonzero entries: - [4, 1] = 2.0 - [1, 2] = 5.0 - [3, 3] = 3.0 - [2, 4] = 4.0 + [4, 1] = 2.0 + [1, 2] = 5.0 + [3, 3] = 3.0 + [2, 4] = 4.0 julia> speye(A) 4×4 sparse matrix with 4 Float64 nonzero entries: - [1, 1] = 1.0 - [2, 2] = 1.0 - [3, 3] = 1.0 - [4, 4] = 1.0 + [1, 1] = 1.0 + [2, 2] = 1.0 + [3, 3] = 1.0 + [4, 4] = 1.0 Note the difference from :func:`spones`\ . @@ -1476,14 +1458,14 @@ dense counterparts. The following functions are specific to sparse arrays. julia> spdiagm(([1,2,3,4],[4,3,2,1]),(-1,1)) 5×5 sparse matrix with 8 Int64 nonzero entries: - [2, 1] = 1 - [1, 2] = 4 - [3, 2] = 2 - [2, 3] = 3 - [4, 3] = 3 - [3, 4] = 2 - [5, 4] = 4 - [4, 5] = 1 + [2, 1] = 1 + [1, 2] = 4 + [3, 2] = 2 + [2, 3] = 3 + [4, 3] = 3 + [3, 4] = 2 + [5, 4] = 4 + [4, 5] = 1 .. function:: sprand([rng],[type],m,[n],p::AbstractFloat,[rfn]) From b9982cdb77d595c4819ce7d4c1d258f3445bcc46 Mon Sep 17 00:00:00 2001 From: kshyatt Date: Fri, 5 Aug 2016 08:46:02 -0700 Subject: [PATCH 22/80] Fixed line in helpdb for doctest (cherry picked from commit fdef48ddafb234166b19a51e1d381552a64d9b80) ref #17797 --- base/docs/helpdb/Base.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 4e6fe7e4fcfd0..7b45975e24423 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -4507,7 +4507,7 @@ julia> deleteat!([6, 5, 4, 3, 2, 1], 1:2:5) julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) ERROR: ArgumentError: indices must be unique and sorted - in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:541 + in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:575 ... ``` """ From 907db9f78e37b4eac43736efa47e38699aef77fd Mon Sep 17 00:00:00 2001 From: kshyatt Date: Sat, 6 Aug 2016 00:20:57 -0700 Subject: [PATCH 23/80] fix docstring wording (cherry picked from commit 6ffe1ab186f5369e18e3ff9a05c3b640454dc1a5) ref #17797 --- base/bitarray.jl | 12 ++++++------ base/sparse/sparsematrix.jl | 2 +- doc/stdlib/arrays.rst | 14 +++++++------- doc/stdlib/collections.rst | 2 +- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/base/bitarray.jl b/base/bitarray.jl index a059248f4ccc5..72272422c7f20 100644 --- a/base/bitarray.jl +++ b/base/bitarray.jl @@ -1426,7 +1426,7 @@ end rol!(dest::BitVector, src::BitVector, i::Integer) -> BitVector Performs a left rotation operation on `src` and puts the result into `dest`. -`i` controls how far to rotate each bit. +`i` controls how far to rotate the bits. """ function rol!(dest::BitVector, src::BitVector, i::Integer) length(dest) == length(src) || throw(ArgumentError("destination and source should be of same size")) @@ -1444,7 +1444,7 @@ end rol!(B::BitVector, i::Integer) -> BitVector Performs a left rotation operation in-place on `B`. -`i` controls how far to rotate each bit. +`i` controls how far to rotate the bits. """ function rol!(B::BitVector, i::Integer) return rol!(B, B, i) @@ -1454,7 +1454,7 @@ end rol(B::BitVector, i::Integer) -> BitVector Performs a left rotation operation, returning a new `BitVector`. -`i` controls how far to rotate each bit. +`i` controls how far to rotate the bits. See also [`rol!`](:func:`rol!`). ```jldoctest @@ -1499,7 +1499,7 @@ end ror!(dest::BitVector, src::BitVector, i::Integer) -> BitVector Performs a right rotation operation on `src` and puts the result into `dest`. -`i` controls how far to rotate each bit. +`i` controls how far to rotate the bits. """ function ror!(dest::BitVector, src::BitVector, i::Integer) length(dest) == length(src) || throw(ArgumentError("destination and source should be of same size")) @@ -1517,7 +1517,7 @@ end ror!(B::BitVector, i::Integer) -> BitVector Performs a right rotation operation in-place on `B`. -`i` controls how far to rotate each bit. +`i` controls how far to rotate the bits. """ function ror!(B::BitVector, i::Integer) return ror!(B, B, i) @@ -1527,7 +1527,7 @@ end ror(B::BitVector, i::Integer) -> BitVector Performs a right rotation operation on `B`, returning a new `BitVector`. -`i` controls how far to rotate each bit. +`i` controls how far to rotate the bits. See also [`ror!`](:func:`ror!`). ```jldoctest diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl index 0e3f2bf68c174..b50e3bef99781 100644 --- a/base/sparse/sparsematrix.jl +++ b/base/sparse/sparsematrix.jl @@ -1258,7 +1258,7 @@ speye(m::Integer, n::Integer) = speye(Float64, m, n) """ speye(S) -Create a sparse identity matrix with the same size as that of `S`. +Create a sparse identity matrix with the same size as `S`. ```jldoctest julia> A = sparse([1,2,3,4],[2,4,3,1],[5.,4.,3.,2.]) diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index 78cdb2e7e44ee..a475fd69b31ad 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -1230,19 +1230,19 @@ to/from the latter via ``Array(bitarray)`` and ``BitArray(array)``, respectively .. Docstring generated from Julia source - Performs a left rotation operation on ``src`` and puts the result into ``dest``\ . ``i`` controls how far to rotate each bit. + Performs a left rotation operation on ``src`` and puts the result into ``dest``\ . ``i`` controls how far to rotate the bits. .. function:: rol!(B::BitVector, i::Integer) -> BitVector .. Docstring generated from Julia source - Performs a left rotation operation in-place on ``B``\ . ``i`` controls how far to rotate each bit. + Performs a left rotation operation in-place on ``B``\ . ``i`` controls how far to rotate the bits. .. function:: rol(B::BitVector, i::Integer) -> BitVector .. Docstring generated from Julia source - Performs a left rotation operation, returning a new ``BitVector``\ . ``i`` controls how far to rotate each bit. See also :func:`rol!`\ . + Performs a left rotation operation, returning a new ``BitVector``\ . ``i`` controls how far to rotate the bits. See also :func:`rol!`\ . .. doctest:: @@ -1282,19 +1282,19 @@ to/from the latter via ``Array(bitarray)`` and ``BitArray(array)``, respectively .. Docstring generated from Julia source - Performs a right rotation operation on ``src`` and puts the result into ``dest``\ . ``i`` controls how far to rotate each bit. + Performs a right rotation operation on ``src`` and puts the result into ``dest``\ . ``i`` controls how far to rotate the bits. .. function:: ror!(B::BitVector, i::Integer) -> BitVector .. Docstring generated from Julia source - Performs a right rotation operation in-place on ``B``\ . ``i`` controls how far to rotate each bit. + Performs a right rotation operation in-place on ``B``\ . ``i`` controls how far to rotate the bits. .. function:: ror(B::BitVector, i::Integer) -> BitVector .. Docstring generated from Julia source - Performs a right rotation operation on ``B``\ , returning a new ``BitVector``\ . ``i`` controls how far to rotate each bit. See also :func:`ror!`\ . + Performs a right rotation operation on ``B``\ , returning a new ``BitVector``\ . ``i`` controls how far to rotate the bits. See also :func:`ror!`\ . .. doctest:: @@ -1428,7 +1428,7 @@ dense counterparts. The following functions are specific to sparse arrays. .. Docstring generated from Julia source - Create a sparse identity matrix with the same size as that of ``S``\ . + Create a sparse identity matrix with the same size as ``S``\ . .. doctest:: diff --git a/doc/stdlib/collections.rst b/doc/stdlib/collections.rst index e37157208b22a..7d3fba881b555 100644 --- a/doc/stdlib/collections.rst +++ b/doc/stdlib/collections.rst @@ -1364,7 +1364,7 @@ Dequeues julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) ERROR: ArgumentError: indices must be unique and sorted - in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:541 + in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:575 ... .. function:: splice!(collection, index, [replacement]) -> item From 203c34d6c4dc3eec0489620ade8094d6da9a376d Mon Sep 17 00:00:00 2001 From: kshyatt Date: Fri, 5 Aug 2016 10:36:34 -0700 Subject: [PATCH 24/80] Move datafmt docs out, update them a bit (cherry picked from commit 16cfe4d71d3d3c77e7941ada1f147fa45a2fb174) ref #17843 --- base/datafmt.jl | 82 ++++++++++++++++++++++++++++++++++ base/docs/helpdb/Base.jl | 93 ++------------------------------------- doc/stdlib/io-network.rst | 4 +- 3 files changed, 87 insertions(+), 92 deletions(-) diff --git a/base/datafmt.jl b/base/datafmt.jl index cc4bd51dcc445..7016d09a3d6d3 100644 --- a/base/datafmt.jl +++ b/base/datafmt.jl @@ -30,14 +30,79 @@ function countlines(io::IO, eol::Char='\n') nl end +""" + readdlm(source, T::Type; options...) + +The columns are assumed to be separated by one or more whitespaces. The end of line +delimiter is taken as `\\n`. +""" readdlm(input, T::Type; opts...) = readdlm(input, invalid_dlm(Char), T, '\n'; opts...) + +""" + readdlm(source, delim::Char, T::Type; options...) + +The end of line delimiter is taken as `\\n`. +""" readdlm(input, dlm::Char, T::Type; opts...) = readdlm(input, dlm, T, '\n'; opts...) +""" + readdlm(source; options...) + +The columns are assumed to be separated by one or more whitespaces. The end of line +delimiter is taken as `\\n`. If all data is numeric, the result will be a numeric array. If +some elements cannot be parsed as numbers, a heterogeneous array of numbers and strings +is returned. +""" readdlm(input; opts...) = readdlm(input, invalid_dlm(Char), '\n'; opts...) + +""" + readdlm(source, delim::Char; options...) + +The end of line delimiter is taken as `\\n`. If all data is numeric, the result will be a +numeric array. If some elements cannot be parsed as numbers, a heterogeneous array of +numbers and strings is returned. +""" readdlm(input, dlm::Char; opts...) = readdlm(input, dlm, '\n'; opts...) +""" + readdlm(source, delim::Char, eol::Char; options...) + +If all data is numeric, the result will be a numeric array. If some elements cannot be +parsed as numbers, a heterogeneous array of numbers and strings is returned. +""" readdlm(input, dlm::Char, eol::Char; opts...) = readdlm_auto(input, dlm, Float64, eol, true; opts...) + +""" + readdlm(source, delim::Char, T::Type, eol::Char; header=false, skipstart=0, skipblanks=true, use_mmap, quotes=true, dims, comments=true, comment_char='#') + +Read a matrix from the source where each line (separated by `eol`) gives one row, with +elements separated by the given delimiter. The source can be a text file, stream or byte +array. Memory mapped files can be used by passing the byte array representation of the +mapped segment as source. + +If `T` is a numeric type, the result is an array of that type, with any non-numeric elements +as `NaN` for floating-point types, or zero. Other useful values of `T` include +`String`, `AbstractString`, and `Any`. + +If `header` is `true`, the first row of data will be read as header and the tuple +`(data_cells, header_cells)` is returned instead of only `data_cells`. + +Specifying `skipstart` will ignore the corresponding number of initial lines from the input. + +If `skipblanks` is `true`, blank lines in the input will be ignored. + +If `use_mmap` is `true`, the file specified by `source` is memory mapped for potential +speedups. Default is `true` except on Windows. On Windows, you may want to specify `true` if +the file is large, and is only read once and not written to. + +If `quotes` is `true`, columns enclosed within double-quote (\") characters are allowed to +contain new lines and column delimiters. Double-quote characters within a quoted field must +be escaped with another double-quote. Specifying `dims` as a tuple of the expected rows and +columns (including header, if any) may speed up reading of large files. If `comments` is +`true`, lines beginning with `comment_char` and text following `comment_char` in any line +are ignored. +""" readdlm(input, dlm::Char, T::Type, eol::Char; opts...) = readdlm_auto(input, dlm, T, eol, false; opts...) @@ -624,7 +689,24 @@ function writedlm(fname::AbstractString, a, dlm; opts...) end end +""" + writedlm(f, A, dl='\\t'; opts) + +Write `A` (a vector, matrix or an iterable collection of iterable rows) as text to `f` +(either a filename string or an `IO` stream) using the given delimiter `delim` (which +defaults to tab, but can be any printable Julia object, typically a `Char` or +`AbstractString`). + +For example, two vectors `x` and `y` of the same length can be written as two columns of +tab-delimited text to `f` by either `writedlm(f, [x y])` or by `writedlm(f, zip(x, y))`. +""" writedlm(io, a; opts...) = writedlm(io, a, '\t'; opts...) + +""" + writecsv(filename, A; opts) + +Equivalent to `writedlm` with `delim` set to comma. +""" writecsv(io, a; opts...) = writedlm(io, a, ','; opts...) show(io::IO, ::MIME"text/csv", a) = writedlm(io, a, ',') diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 7b45975e24423..67812fa2c1f2d 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -10,17 +10,11 @@ Raises a `SystemError` for `errno` with the descriptive string `sysfunc` if `ift systemerror """ - writedlm(f, A, delim='\\t') + digamma(x) -Write `A` (a vector, matrix or an iterable collection of iterable rows) as text to `f` -(either a filename string or an `IO` stream) using the given delimiter `delim` (which -defaults to tab, but can be any printable Julia object, typically a `Char` or -`AbstractString`). - -For example, two vectors `x` and `y` of the same length can be written as two columns of -tab-delimited text to `f` by either `writedlm(f, [x y])` or by `writedlm(f, zip(x, y))`. +Compute the digamma function of `x` (the logarithmic derivative of `gamma(x)`) """ -writedlm +digamma """ fill!(A, x) @@ -2236,13 +2230,6 @@ Largest integer less than or equal to `x/y`. """ fld -""" - writecsv(filename, A) - -Equivalent to `writedlm` with `delim` set to comma. -""" -writecsv - """ withenv(f::Function, kv::Pair...) @@ -2860,80 +2847,6 @@ second variant. """ popdisplay -""" - readdlm(source, delim::Char, T::Type, eol::Char; header=false, skipstart=0, skipblanks=true, use_mmap, quotes=true, dims, comments=true, comment_char='#') - -Read a matrix from the source where each line (separated by `eol`) gives one row, with -elements separated by the given delimiter. The source can be a text file, stream or byte -array. Memory mapped files can be used by passing the byte array representation of the -mapped segment as source. - -If `T` is a numeric type, the result is an array of that type, with any non-numeric elements -as `NaN` for floating-point types, or zero. Other useful values of `T` include -`String`, `AbstractString`, and `Any`. - -If `header` is `true`, the first row of data will be read as header and the tuple -`(data_cells, header_cells)` is returned instead of only `data_cells`. - -Specifying `skipstart` will ignore the corresponding number of initial lines from the input. - -If `skipblanks` is `true`, blank lines in the input will be ignored. - -If `use_mmap` is `true`, the file specified by `source` is memory mapped for potential -speedups. Default is `true` except on Windows. On Windows, you may want to specify `true` if -the file is large, and is only read once and not written to. - -If `quotes` is `true`, columns enclosed within double-quote (\") characters are allowed to -contain new lines and column delimiters. Double-quote characters within a quoted field must -be escaped with another double-quote. Specifying `dims` as a tuple of the expected rows and -columns (including header, if any) may speed up reading of large files. If `comments` is -`true`, lines beginning with `comment_char` and text following `comment_char` in any line -are ignored. -""" -readdlm(source, delim, T, eol) - -""" - readdlm(source, delim::Char, eol::Char; options...) - -If all data is numeric, the result will be a numeric array. If some elements cannot be -parsed as numbers, a heterogeneous array of numbers and strings is returned. -""" -readdlm(source, delim::Char, eol::Char) - -""" - readdlm(source, delim::Char, T::Type; options...) - -The end of line delimiter is taken as `\\n`. -""" -readdlm(source, delim::Char, T::Type) - -""" - readdlm(source, delim::Char; options...) - -The end of line delimiter is taken as `\\n`. If all data is numeric, the result will be a -numeric array. If some elements cannot be parsed as numbers, a heterogeneous array of -numbers and strings is returned. -""" -readdlm(source, delim::Char) - -""" - readdlm(source, T::Type; options...) - -The columns are assumed to be separated by one or more whitespaces. The end of line -delimiter is taken as `\\n`. -""" -readdlm(source, T::Type) - -""" - readdlm(source; options...) - -The columns are assumed to be separated by one or more whitespaces. The end of line -delimiter is taken as `\\n`. If all data is numeric, the result will be a numeric array. If -some elements cannot be parsed as numbers, a heterogeneous array of numbers and strings -is returned. -""" -readdlm(source) - """ filesize(path...) diff --git a/doc/stdlib/io-network.rst b/doc/stdlib/io-network.rst index 158560b2c9ba3..6f4737b133498 100644 --- a/doc/stdlib/io-network.rst +++ b/doc/stdlib/io-network.rst @@ -624,7 +624,7 @@ Text I/O The columns are assumed to be separated by one or more whitespaces. The end of line delimiter is taken as ``\n``\ . If all data is numeric, the result will be a numeric array. If some elements cannot be parsed as numbers, a heterogeneous array of numbers and strings is returned. -.. function:: writedlm(f, A, delim='\\t') +.. function:: writedlm(f, A, dl='\\t'; opts) .. Docstring generated from Julia source @@ -638,7 +638,7 @@ Text I/O Equivalent to ``readdlm`` with ``delim`` set to comma. -.. function:: writecsv(filename, A) +.. function:: writecsv(filename, A; opts) .. Docstring generated from Julia source From bb4053856a9e747de157cbc58807f1a7b73603ef Mon Sep 17 00:00:00 2001 From: Katharine Hyatt Date: Sat, 6 Aug 2016 19:14:21 -0700 Subject: [PATCH 25/80] Update some function sigs for tasks and events, clear out HelpDB (#17846) (cherry picked from commit 6ba3b976dd5da09a7ed55e0bd531180a26ae4261) --- base/docs/helpdb/Base.jl | 123 --------------------------------------- base/event.jl | 47 +++++++++++++++ base/task.jl | 53 ++++++++++++++++- doc/stdlib/parallel.rst | 24 ++++---- 4 files changed, 111 insertions(+), 136 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 67812fa2c1f2d..82604311cd1cc 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -54,18 +54,6 @@ Subtype operator, equivalent to `issubtype(T1,T2)`. """ Base.:(<:) -""" - schedule(t::Task, [val]; error=false) - -Add a task to the scheduler's queue. This causes the task to run constantly when the system -is otherwise idle, unless the task performs a blocking operation such as `wait`. - -If a second argument is provided, it will be passed to the task (via the return value of -`yieldto`) when it runs again. If `error` is `true`, the value is raised as an exception in -the woken task. -""" -schedule - """ takebuf_array(b::IOBuffer) @@ -258,29 +246,6 @@ Returns `true` if `path` is a regular file, `false` otherwise. """ isfile -""" - task_local_storage(symbol) - -Look up the value of a symbol in the current task's task-local storage. -""" -task_local_storage(symbol) - -""" - task_local_storage(symbol, value) - -Assign a value to a symbol in the current task's task-local storage. -""" -task_local_storage(symbol, value) - -""" - task_local_storage(body, symbol, value) - -Call the function `body` with a modified task-local storage, in which `value` is assigned to -`symbol`; the previous value of `symbol`, or lack thereof, is restored afterwards. Useful -for emulating dynamic scoping. -""" -task_local_storage(body, symbol, value) - """ diff(A, [dim]) @@ -1107,13 +1072,6 @@ Convert all arguments to their common promotion type (if any), and return them a """ promote -""" - @schedule - -Wrap an expression in a `Task` and add it to the local machine's scheduler queue. -""" -:@schedule - """ gradient(F, [h]) @@ -1465,15 +1423,6 @@ Dict{String,Float64} with 3 entries: """ merge -""" - yield() - -Switch to the scheduler to allow another scheduled task to run. A task that calls this -function is still runnable, and will be restarted immediately if there are no other runnable -tasks. -""" -yield - """ transpose!(dest,src) @@ -2215,14 +2164,6 @@ Compute the LU factorization of `A`, such that `A[p,:] = L*U`. """ lu -""" - @task - -Wrap an expression in a `Task` without executing it, and return the `Task`. This only -creates a task, and does not run it. -""" -:@task - """ fld(x, y) @@ -2770,16 +2711,6 @@ Compute the inverse error function of a real `x`, defined by ``\\operatorname{er """ erfinv -""" - @async - -Like `@schedule`, `@async` wraps an expression in a `Task` and adds it to the local -machine's scheduler queue. Additionally it adds the task to the set of items that the -nearest enclosing `@sync` waits for. `@async` also wraps the expression in a `let x=x, y=y, ...` -block to create a new scope with copies of all variables referenced in the expression. -""" -:@async - """ readdir([dir]) -> Vector{String} @@ -4041,13 +3972,6 @@ Like `selectperm`, but accepts a preallocated index vector `ix`. If `initialized """ selectperm! -""" - istaskdone(task) -> Bool - -Tell whether a task has exited. -""" -istaskdone - """ .>(x, y) @@ -4187,15 +4111,6 @@ Determine whether a stream is read-only. """ isreadonly -""" - notify(condition, val=nothing; all=true, error=false) - -Wake up tasks waiting for a condition, passing them `val`. If `all` is `true` (the default), -all waiting tasks are woken, otherwise only one is. If `error` is `true`, the passed value -is raised as an exception in the woken tasks. -""" -notify - """ view(A, inds...) @@ -4868,15 +4783,6 @@ Type conversion cannot be done exactly. """ InexactError -""" - @sync - -Wait until all dynamically-enclosed uses of `@async`, `@spawn`, `@spawnat` and `@parallel` -are complete. All exceptions thrown by enclosed async operations are collected and thrown as -a `CompositeException`. -""" -:@sync - """ typemax(T) @@ -6723,18 +6629,6 @@ base64-encoded string. """ base64encode -""" - Condition() - -Create an edge-triggered event source that tasks can wait for. Tasks that call `wait` on a -`Condition` are suspended and queued. Tasks are woken up when `notify` is later called on -the `Condition`. Edge triggering means that only tasks waiting at the time `notify` is -called can be woken up. For level-triggered notifications, you must keep extra state to keep -track of whether a notification has happened. The `Channel` type does this, and so can be -used for level-triggered events. -""" -Condition - """ filt!(out, b, a, x, [si]) @@ -7912,16 +7806,6 @@ Bitwise or. """ Base.:(|) -""" - yieldto(task, arg = nothing) - -Switch to the given task. The first time a task is switched to, the task's function is -called with no arguments. On subsequent switches, `arg` is returned from the task's last -call to `yieldto`. This is a low-level call that only switches tasks, not considering states -or scheduling in any way. Its use is discouraged. -""" -yieldto - """ readandwrite(command) @@ -8072,13 +7956,6 @@ Compute the Dawson function (scaled imaginary error function) of `x`, defined by """ dawson -""" - current_task() - -Get the currently running `Task`. -""" -current_task - """ randjump(r::MersenneTwister, jumps, [jumppoly]) -> Vector{MersenneTwister} diff --git a/base/event.jl b/base/event.jl index e91849b40b11f..67beaa308a867 100644 --- a/base/event.jl +++ b/base/event.jl @@ -2,6 +2,16 @@ ## condition variables +""" + Condition() + +Create an edge-triggered event source that tasks can wait for. Tasks that call `wait` on a +`Condition` are suspended and queued. Tasks are woken up when `notify` is later called on +the `Condition`. Edge triggering means that only tasks waiting at the time `notify` is +called can be woken up. For level-triggered notifications, you must keep extra state to keep +track of whether a notification has happened. The [`Channel`](:class:`Channel`) type does +this, and so can be used for level-triggered events. +""" type Condition waitq::Vector{Any} @@ -21,6 +31,13 @@ function wait(c::Condition) end end +""" + notify(condition, val=nothing; all=true, error=false) + +Wake up tasks waiting for a condition, passing them `val`. If `all` is `true` (the default), +all waiting tasks are woken, otherwise only one is. If `error` is `true`, the passed value +is raised as an exception in the woken tasks. +""" notify(c::Condition, arg::ANY=nothing; all=true, error=false) = notify(c, arg, all, error) function notify(c::Condition, arg, all, error) if all @@ -42,6 +59,11 @@ notify1_error(c::Condition, err) = notify(c, err, error=true, all=false) # schedule an expression to run asynchronously, with minimal ceremony +""" + @schedule + +Wrap an expression in a `Task` and add it to the local machine's scheduler queue. +""" macro schedule(expr) expr = :(()->($expr)) :(enq_work(Task($(esc(expr))))) @@ -61,6 +83,16 @@ end schedule(t::Task) = enq_work(t) +""" + schedule(t::Task, [val]; error=false) + +Add a task to the scheduler's queue. This causes the task to run constantly when the system +is otherwise idle, unless the task performs a blocking operation such as `wait`. + +If a second argument `val` is provided, it will be passed to the task (via the return value of +`yieldto`) when it runs again. If `error` is `true`, the value is raised as an exception in +the woken task. +""" function schedule(t::Task, arg; error=false) # schedule a task to be (re)started with the given value or exception if error @@ -84,8 +116,23 @@ function schedule_and_wait(t::Task, v=nothing) return wait() end +""" + yield() + +Switch to the scheduler to allow another scheduled task to run. A task that calls this +function is still runnable, and will be restarted immediately if there are no other runnable +tasks. +""" yield() = (enq_work(current_task()); wait()) +""" + yieldto(task, arg = nothing) + +Switch to the given task. The first time a task is switched to, the task's function is +called with no arguments. On subsequent switches, `arg` is returned from the task's last +call to `yieldto`. This is a low-level call that only switches tasks, not considering states +or scheduling in any way. Its use is discouraged. +""" yieldto(t::Task, x::ANY = nothing) = ccall(:jl_switchto, Any, (Any, Any), t, x) # yield to a task, throwing an exception in it diff --git a/base/task.jl b/base/task.jl index 148b77861cc58..cc06115a23554 100644 --- a/base/task.jl +++ b/base/task.jl @@ -50,17 +50,34 @@ function show(io::IO, t::Task) print(io, "Task ($(t.state)) @0x$(hex(convert(UInt, pointer_from_objref(t)), Sys.WORD_SIZE>>2))") end +""" + @task + +Wrap an expression in a [`Task`](:class:`Task`) without executing it, and return the [`Task`](:class:`Task`). This only +creates a task, and does not run it. +""" macro task(ex) :(Task(()->$(esc(ex)))) end +""" + current_task() + +Get the currently running [`Task`](:class:`Task`). +""" current_task() = ccall(:jl_get_current_task, Ref{Task}, ()) + +""" + istaskdone(task) -> Bool + +Determine whether a task has exited. +""" istaskdone(t::Task) = ((t.state == :done) | (t.state == :failed)) """ istaskstarted(task) -> Bool -Tell whether a task has started executing. +Determine whether a task has started executing. """ istaskstarted(t::Task) = ccall(:jl_is_task_started, Cint, (Any,), t) != 0 @@ -71,9 +88,28 @@ function get_task_tls(t::Task) end (t.storage)::ObjectIdDict end + +""" + task_local_storage(key) + +Look up the value of a key in the current task's task-local storage. +""" task_local_storage(key) = task_local_storage()[key] + +""" + task_local_storage(key, value) + +Assign a value to a key in the current task's task-local storage. +""" task_local_storage(key, val) = (task_local_storage()[key] = val) +""" + task_local_storage(body, key, value) + +Call the function `body` with a modified task-local storage, in which `value` is assigned to +`key`; the previous value of `key`, or lack thereof, is restored afterwards. Useful +for emulating dynamic scoping. +""" function task_local_storage(body::Function, key, val) tls = task_local_storage() hadkey = haskey(tls,key) @@ -277,6 +313,13 @@ function sync_end() nothing end +""" + @sync + +Wait until all dynamically-enclosed uses of `@async`, `@spawn`, `@spawnat` and `@parallel` +are complete. All exceptions thrown by enclosed async operations are collected and thrown as +a `CompositeException`. +""" macro sync(block) quote sync_begin() @@ -305,6 +348,14 @@ function async_run_thunk(thunk) t end +""" + @async + +Like `@schedule`, `@async` wraps an expression in a `Task` and adds it to the local +machine's scheduler queue. Additionally it adds the task to the set of items that the +nearest enclosing `@sync` waits for. `@async` also wraps the expression in a `let x=x, y=y, ...` +block to create a new scope with copies of all variables referenced in the expression. +""" macro async(expr) expr = localize_vars(esc(:(()->($expr))), false) :(async_run_thunk($expr)) diff --git a/doc/stdlib/parallel.rst b/doc/stdlib/parallel.rst index f7759523d99fb..9aeaeb22d096f 100644 --- a/doc/stdlib/parallel.rst +++ b/doc/stdlib/parallel.rst @@ -23,19 +23,19 @@ Tasks .. Docstring generated from Julia source - Get the currently running ``Task``\ . + Get the currently running :class:`Task`\ . .. function:: istaskdone(task) -> Bool .. Docstring generated from Julia source - Tell whether a task has exited. + Determine whether a task has exited. .. function:: istaskstarted(task) -> Bool .. Docstring generated from Julia source - Tell whether a task has started executing. + Determine whether a task has started executing. .. function:: consume(task, values...) @@ -55,29 +55,29 @@ Tasks Switch to the scheduler to allow another scheduled task to run. A task that calls this function is still runnable, and will be restarted immediately if there are no other runnable tasks. -.. function:: task_local_storage(symbol) +.. function:: task_local_storage(key) .. Docstring generated from Julia source - Look up the value of a symbol in the current task's task-local storage. + Look up the value of a key in the current task's task-local storage. -.. function:: task_local_storage(symbol, value) +.. function:: task_local_storage(key, value) .. Docstring generated from Julia source - Assign a value to a symbol in the current task's task-local storage. + Assign a value to a key in the current task's task-local storage. -.. function:: task_local_storage(body, symbol, value) +.. function:: task_local_storage(body, key, value) .. Docstring generated from Julia source - Call the function ``body`` with a modified task-local storage, in which ``value`` is assigned to ``symbol``\ ; the previous value of ``symbol``\ , or lack thereof, is restored afterwards. Useful for emulating dynamic scoping. + Call the function ``body`` with a modified task-local storage, in which ``value`` is assigned to ``key``\ ; the previous value of ``key``\ , or lack thereof, is restored afterwards. Useful for emulating dynamic scoping. .. function:: Condition() .. Docstring generated from Julia source - Create an edge-triggered event source that tasks can wait for. Tasks that call ``wait`` on a ``Condition`` are suspended and queued. Tasks are woken up when ``notify`` is later called on the ``Condition``\ . Edge triggering means that only tasks waiting at the time ``notify`` is called can be woken up. For level-triggered notifications, you must keep extra state to keep track of whether a notification has happened. The ``Channel`` type does this, and so can be used for level-triggered events. + Create an edge-triggered event source that tasks can wait for. Tasks that call ``wait`` on a ``Condition`` are suspended and queued. Tasks are woken up when ``notify`` is later called on the ``Condition``\ . Edge triggering means that only tasks waiting at the time ``notify`` is called can be woken up. For level-triggered notifications, you must keep extra state to keep track of whether a notification has happened. The :class:`Channel` type does this, and so can be used for level-triggered events. .. function:: notify(condition, val=nothing; all=true, error=false) @@ -91,7 +91,7 @@ Tasks Add a task to the scheduler's queue. This causes the task to run constantly when the system is otherwise idle, unless the task performs a blocking operation such as ``wait``\ . - If a second argument is provided, it will be passed to the task (via the return value of ``yieldto``\ ) when it runs again. If ``error`` is ``true``\ , the value is raised as an exception in the woken task. + If a second argument ``val`` is provided, it will be passed to the task (via the return value of ``yieldto``\ ) when it runs again. If ``error`` is ``true``\ , the value is raised as an exception in the woken task. .. function:: @schedule @@ -103,7 +103,7 @@ Tasks .. Docstring generated from Julia source - Wrap an expression in a ``Task`` without executing it, and return the ``Task``\ . This only creates a task, and does not run it. + Wrap an expression in a :class:`Task` without executing it, and return the :class:`Task`\ . This only creates a task, and does not run it. .. function:: sleep(seconds) From 87f160cd98796c8504527488d90a4a96a63739eb Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" Date: Sat, 6 Aug 2016 20:32:42 +0530 Subject: [PATCH 26/80] Bump openlibm. (cherry picked from commit 03e92621aa93431ea78aa0d511427dce032035bf) ref #17864 --- .../md5 | 1 + .../sha512 | 1 + .../md5 | 1 - .../sha512 | 1 - deps/openlibm.version | 4 ++-- 5 files changed, 4 insertions(+), 4 deletions(-) create mode 100644 deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/md5 create mode 100644 deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/sha512 delete mode 100644 deps/checksums/openlibm-e2fc5dd2f86f1e1dc47e8fa153b6a7b776d53ab5.tar.gz/md5 delete mode 100644 deps/checksums/openlibm-e2fc5dd2f86f1e1dc47e8fa153b6a7b776d53ab5.tar.gz/sha512 diff --git a/deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/md5 b/deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/md5 new file mode 100644 index 0000000000000..8e7d4cc7c4029 --- /dev/null +++ b/deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/md5 @@ -0,0 +1 @@ +3b852052db9052a3668122449ce1f82c diff --git a/deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/sha512 b/deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/sha512 new file mode 100644 index 0000000000000..a840095574b1a --- /dev/null +++ b/deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/sha512 @@ -0,0 +1 @@ +7125800186428a8aefea3030ff1d35f4169c29cb3ae30c2bd7965bd5e66f0958b8b33fcd2e5a30b012ecececd09ce3ce0195b5a8fc9066efd31e124cc32ec2c0 diff --git a/deps/checksums/openlibm-e2fc5dd2f86f1e1dc47e8fa153b6a7b776d53ab5.tar.gz/md5 b/deps/checksums/openlibm-e2fc5dd2f86f1e1dc47e8fa153b6a7b776d53ab5.tar.gz/md5 deleted file mode 100644 index 5b6d306e94650..0000000000000 --- a/deps/checksums/openlibm-e2fc5dd2f86f1e1dc47e8fa153b6a7b776d53ab5.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -7d15bdbd8ba248ab8ff6a120c7806cf0 diff --git a/deps/checksums/openlibm-e2fc5dd2f86f1e1dc47e8fa153b6a7b776d53ab5.tar.gz/sha512 b/deps/checksums/openlibm-e2fc5dd2f86f1e1dc47e8fa153b6a7b776d53ab5.tar.gz/sha512 deleted file mode 100644 index f5322da6ded37..0000000000000 --- a/deps/checksums/openlibm-e2fc5dd2f86f1e1dc47e8fa153b6a7b776d53ab5.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -9977fcc79e54c86ec8373f73f98f95dc7c4b20ca28884998fbb7f0aa3759ff904ee326cfcbbaf0d000de9161e8d1511e71440b259a95517742b803ef8be32bdc diff --git a/deps/openlibm.version b/deps/openlibm.version index 4e15529fd3583..ceb185d9bc9de 100644 --- a/deps/openlibm.version +++ b/deps/openlibm.version @@ -1,2 +1,2 @@ -OPENLIBM_BRANCH=v0.5.1 -OPENLIBM_SHA1=e2fc5dd2f86f1e1dc47e8fa153b6a7b776d53ab5 +OPENLIBM_BRANCH=v0.5.2 +OPENLIBM_SHA1=0fa599cce845be67ed1729d9753f67e366c37b96 From eb2f488e1ebab9fe65853bac8940e49c3f14ce14 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Sun, 7 Aug 2016 06:45:29 +0000 Subject: [PATCH 27/80] Test LibGit2 SSH authentication (#17651) (cherry picked from commit 7f074e94d70bb6cd97e981cd7b4867db8715f9d1) --- base/libgit2/callbacks.jl | 17 ++-- base/libgit2/error.jl | 1 + test/TestHelpers.jl | 27 ++++++ test/choosetests.jl | 2 +- test/libgit2.jl | 177 ++++++++++++++++++++++++++++++++++++++ test/repl.jl | 51 ++++------- 6 files changed, 231 insertions(+), 44 deletions(-) diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index 81aa56c56d78a..0143125c3c2f7 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -64,9 +64,7 @@ function authenticate_ssh(creds::SSHCredentials, libgit2credptr::Ptr{Ptr{Void}}, else keydefpath = creds.prvkey # check if credentials were already used keydefpath === nothing && (keydefpath = "") - if !isempty(keydefpath) && !isusedcreds - keydefpath # use cached value - else + if isempty(keydefpath) || isusedcreds defaultkeydefpath = joinpath(homedir(),".ssh","id_rsa") if isempty(keydefpath) && isfile(defaultkeydefpath) keydefpath = defaultkeydefpath @@ -75,6 +73,7 @@ function authenticate_ssh(creds::SSHCredentials, libgit2credptr::Ptr{Ptr{Void}}, prompt("Private key location for '$schema$username@$host'", default=keydefpath) end end + keydefpath end # If the private key changed, invalidate the cached public key @@ -87,18 +86,16 @@ function authenticate_ssh(creds::SSHCredentials, libgit2credptr::Ptr{Ptr{Void}}, ENV["SSH_PUB_KEY_PATH"] else keydefpath = creds.pubkey # check if credentials were already used - if keydefpath !== nothing && !isusedcreds - keydefpath # use cached value - else - if keydefpath === nothing || isempty(keydefpath) + keydefpath === nothing && (keydefpath = "") + if isempty(keydefpath) || isusedcreds + if isempty(keydefpath) keydefpath = privatekey*".pub" end - if isfile(keydefpath) - keydefpath - else + if !isfile(keydefpath) prompt("Public key location for '$schema$username@$host'", default=keydefpath) end end + keydefpath end creds.pubkey = publickey # save credentials diff --git a/base/libgit2/error.jl b/base/libgit2/error.jl index 823b4e3907ded..1d6c904a62e57 100644 --- a/base/libgit2/error.jl +++ b/base/libgit2/error.jl @@ -23,6 +23,7 @@ export GitError ECERTIFICATE = Cint(-17), # server certificate is invalid EAPPLIED = Cint(-18), # patch/merge has already been applied EPEEL = Cint(-19), # the requested peel operation is not possible + EEOF = Cint(-20), # Unexpted EOF PASSTHROUGH = Cint(-30), # internal only ITEROVER = Cint(-31)) # signals end of iteration diff --git a/test/TestHelpers.jl b/test/TestHelpers.jl index 609962e68e389..b57b0d10f77ee 100644 --- a/test/TestHelpers.jl +++ b/test/TestHelpers.jl @@ -16,4 +16,31 @@ Base.Terminals.hascolor(t::FakeTerminal) = t.hascolor Base.Terminals.raw!(t::FakeTerminal, raw::Bool) = t.raw = raw Base.Terminals.size(t::FakeTerminal) = (24, 80) +function open_fake_pty() + const O_RDWR = Base.Filesystem.JL_O_RDWR + const O_NOCTTY = Base.Filesystem.JL_O_NOCTTY + + fdm = ccall(:posix_openpt, Cint, (Cint,), O_RDWR|O_NOCTTY) + fdm == -1 && error("Failed to open PTY master") + rc = ccall(:grantpt, Cint, (Cint,), fdm) + rc != 0 && error("grantpt failed") + rc = ccall(:unlockpt, Cint, (Cint,), fdm) + rc != 0 && error("unlockpt") + + fds = ccall(:open, Cint, (Ptr{UInt8}, Cint), + ccall(:ptsname, Ptr{UInt8}, (Cint,), fdm), O_RDWR|O_NOCTTY) + + # slave + slave = RawFD(fds) + master = Base.TTY(RawFD(fdm); readable = true) + slave, master +end + +function with_fake_pty(f) + slave, master = open_fake_pty() + f(slave, master) + ccall(:close,Cint,(Cint,),slave) # XXX: this causes the kernel to throw away all unread data on the pty + close(master) +end + end diff --git a/test/choosetests.jl b/test/choosetests.jl index 9484d0e394f7c..d02ed1dcf3cf0 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -80,7 +80,7 @@ function choosetests(choices = []) prepend!(tests, linalgtests) end - net_required_for = ["socket", "parallel"] + net_required_for = ["socket", "parallel", "libgit2"] net_on = true try getipaddr() diff --git a/test/libgit2.jl b/test/libgit2.jl index c5ba7a9082e9d..a0466c21c7e15 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -2,6 +2,9 @@ #@testset "libgit2" begin +isdefined(:TestHelpers) || include(joinpath(dirname(@__FILE__), "TestHelpers.jl")) +using TestHelpers + const LIBGIT2_MIN_VER = v"0.23.0" ######### @@ -567,6 +570,180 @@ mktempdir() do dir @test creds.user == creds_user @test creds.pass == creds_pass #end + + #@testset "SSH" begin + sshd_command = "" + ssh_repo = joinpath(dir, "Example.SSH") + if !is_windows() + try + # SSHD needs to be executed by its full absolute path + sshd_command = strip(readstring(`which sshd`)) + catch + warn("Skipping SSH tests (Are `which` and `sshd` installed?)") + end + end + if !isempty(sshd_command) + mktempdir() do fakehomedir + mkdir(joinpath(fakehomedir,".ssh")) + # Unsetting the SSH agent serves two purposes. First, we make + # sure that we don't accidentally pick up an existing agent, + # and second we test that we fall back to using a key file + # if the agent isn't present. + withenv("HOME"=>fakehomedir,"SSH_AUTH_SOCK"=>nothing) do + # Generate user file, first an unencrypted one + wait(spawn(`ssh-keygen -N "" -C juliatest@localhost -f $fakehomedir/.ssh/id_rsa`)) + + # Generate host keys + wait(spawn(`ssh-keygen -f $fakehomedir/ssh_host_rsa_key -N '' -t rsa`)) + wait(spawn(`ssh-keygen -f $fakehomedir/ssh_host_dsa_key -N '' -t dsa`)) + + our_ssh_port = rand(13000:14000) # Chosen arbitrarily + + key_option = "AuthorizedKeysFile $fakehomedir/.ssh/id_rsa.pub" + pidfile_option = "PidFile $fakehomedir/sshd.pid" + sshp = agentp = nothing + logfile = tempname() + ssh_debug = false + function spawn_sshd() + debug_flags = ssh_debug ? `-d -d` : `` + _p = open(logfile, "a") do logfilestream + spawn(pipeline(pipeline(`$sshd_command + -e -f /dev/null $debug_flags + -h $fakehomedir/ssh_host_rsa_key + -h $fakehomedir/ssh_host_dsa_key -p $our_ssh_port + -o $pidfile_option + -o 'Protocol 2' + -o $key_option + -o 'UsePrivilegeSeparation no' + -o 'StrictModes no'`,STDOUT),stderr=logfilestream)) + end + # Give the SSH server 5 seconds to start up + yield(); sleep(5) + _p + end + sshp = spawn_sshd() + + TIOCSCTTY_str = "ccall(:ioctl, Void, (Cint, Cint, Int64), 0, + (is_bsd() || is_apple()) ? 0x20007461 : is_linux() ? 0x540E : + error(\"Fill in TIOCSCTTY for this OS here\"), 0)" + + # To fail rather than hang + function killer_task(p, master) + @async begin + sleep(10) + kill(p) + if isopen(master) + nb_available(master) > 0 && + write(logfile, + readavailable(master)) + close(master) + end + end + end + + try + function try_clone(challenges = []) + cmd = """ + repo = nothing + try + $TIOCSCTTY_str + reponame = "ssh://$(ENV["USER"])@localhost:$our_ssh_port$cache_repo" + repo = LibGit2.clone(reponame, "$ssh_repo") + catch err + open("$logfile","a") do f + println(f,"HOME: ",ENV["HOME"]) + println(f, err) + end + finally + finalize(repo) + end + """ + # We try to be helpful by desparately looking for + # a way to prompt the password interactively. Pretend + # to be a TTY to suppress those shenanigans. Further, we + # need to detach and change the controlling terminal with + # TIOCSCTTY, since getpass opens the controlling terminal + TestHelpers.with_fake_pty() do slave, master + err = Base.Pipe() + let p = spawn(detach( + `$(Base.julia_cmd()) --startup-file=no -e $cmd`),slave,slave,STDERR) + killer_task(p, master) + for (challenge, response) in challenges + readuntil(master, challenge) + sleep(1) + print(master, response) + end + sleep(2) + wait(p) + close(master) + end + end + @test isfile(joinpath(ssh_repo,"testfile")) + rm(ssh_repo, recursive = true) + end + + # Should use the default files, no interaction required. + try_clone() + ssh_debug && (kill(sshp); sshp = spawn_sshd()) + + # Ok, now encrypt the file and test with that (this also + # makes sure that we don't accidentally fall back to the + # unencrypted version) + wait(spawn(`ssh-keygen -p -N "xxxxx" -f $fakehomedir/.ssh/id_rsa`)) + + # Try with the encrypted file. Needs a password. + try_clone(["Passphrase"=>"xxxxx\r\n"]) + ssh_debug && (kill(sshp); sshp = spawn_sshd()) + + # Move the file. It should now ask for the location and + # then the passphrase + mv("$fakehomedir/.ssh/id_rsa","$fakehomedir/.ssh/id_rsa2") + cp("$fakehomedir/.ssh/id_rsa.pub","$fakehomedir/.ssh/id_rsa2.pub") + try_clone(["location"=>"$fakehomedir/.ssh/id_rsa2\n", + "Passphrase"=>"xxxxx\n"]) + mv("$fakehomedir/.ssh/id_rsa2","$fakehomedir/.ssh/id_rsa") + rm("$fakehomedir/.ssh/id_rsa2.pub") + + # Ok, now start an agent + agent_sock = tempname() + agentp = spawn(`ssh-agent -a $agent_sock -d`) + while stat(agent_sock).mode == 0 # Wait until the agent is started + sleep(1) + end + + # fake pty is required for the same reason as in try_clone + # above + withenv("SSH_AUTH_SOCK" => agent_sock) do + TestHelpers.with_fake_pty() do slave, master + cmd = """ + $TIOCSCTTY_str + run(pipeline(`ssh-add $fakehomedir/.ssh/id_rsa`, + stderr = DevNull)) + """ + addp = spawn(detach(`$(Base.julia_cmd()) --startup-file=no -e $cmd`), + slave, slave, STDERR) + killer_task(addp, master) + sleep(2) + write(master, "xxxxx\n") + wait(addp) + end + + # Should now use the agent + try_clone() + end + catch err + println("SSHD logfile contents follows:") + println(readstring(logfile)) + rethrow(err) + finally + rm(logfile) + sshp !== nothing && kill(sshp) + agentp !== nothing && kill(agentp) + end + end + end + end + #end end #end diff --git a/test/repl.jl b/test/repl.jl index f75411b165b58..09653bcb8422a 100644 --- a/test/repl.jl +++ b/test/repl.jl @@ -444,40 +444,25 @@ let exename = Base.julia_cmd() # Test REPL in dumb mode if !is_windows() - const O_RDWR = Base.Filesystem.JL_O_RDWR - const O_NOCTTY = Base.Filesystem.JL_O_NOCTTY - - fdm = ccall(:posix_openpt, Cint, (Cint,), O_RDWR|O_NOCTTY) - fdm == -1 && error("Failed to open PTY master") - rc = ccall(:grantpt, Cint, (Cint,), fdm) - rc != 0 && error("grantpt failed") - rc = ccall(:unlockpt, Cint, (Cint,), fdm) - rc != 0 && error("unlockpt") - - fds = ccall(:open, Cint, (Ptr{UInt8}, Cint), - ccall(:ptsname, Ptr{UInt8}, (Cint,), fdm), O_RDWR|O_NOCTTY) - - # slave - slave = RawFD(fds) - master = Base.TTY(RawFD(fdm); readable = true) - - nENV = copy(ENV) - nENV["TERM"] = "dumb" - p = spawn(setenv(`$exename --startup-file=no --quiet`,nENV),slave,slave,slave) - output = readuntil(master,"julia> ") - if ccall(:jl_running_on_valgrind,Cint,()) == 0 - # If --trace-children=yes is passed to valgrind, we will get a - # valgrind banner here, not just the prompt. - @test output == "julia> " + TestHelpers.with_fake_pty() do slave, master + + nENV = copy(ENV) + nENV["TERM"] = "dumb" + p = spawn(setenv(`$exename --startup-file=no --quiet`,nENV),slave,slave,slave) + output = readuntil(master,"julia> ") + if ccall(:jl_running_on_valgrind,Cint,()) == 0 + # If --trace-children=yes is passed to valgrind, we will get a + # valgrind banner here, not just the prompt. + @test output == "julia> " + end + write(master,"1\nquit()\n") + + wait(p) + output = readuntil(master,' ') + @test output == "1\r\nquit()\r\n1\r\n\r\njulia> " + @test nb_available(master) == 0 + end - write(master,"1\nquit()\n") - - wait(p) - output = readuntil(master,' ') - @test output == "1\r\nquit()\r\n1\r\n\r\njulia> " - @test nb_available(master) == 0 - ccall(:close,Cint,(Cint,),fds) # XXX: this causes the kernel to throw away all unread data on the pty - close(master) end # Test stream mode From 8f06a6cc73844eb56ff219080ae810e973189707 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Wed, 10 Aug 2016 22:04:06 -0700 Subject: [PATCH 28/80] Temporarily disable SSH tests until we can get them working on buildbots ref https://github.com/JuliaLang/julia/pull/17651#issuecomment-238211150 --- test/libgit2.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/libgit2.jl b/test/libgit2.jl index a0466c21c7e15..db7d4924688c9 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -571,7 +571,7 @@ mktempdir() do dir @test creds.pass == creds_pass #end - #@testset "SSH" begin + #= temporarily disabled until working on the buildbots, ref https://github.com/JuliaLang/julia/pull/17651#issuecomment-238211150 sshd_command = "" ssh_repo = joinpath(dir, "Example.SSH") if !is_windows() @@ -743,7 +743,7 @@ mktempdir() do dir end end end - #end + =# end #end From 7a5e180d1c7f6a24d5016c06b1f84caa0c5e6843 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Sun, 7 Aug 2016 02:03:01 -0500 Subject: [PATCH 29/80] Fix copy(region::Tuple) depwarn in FFTW (#17840) (cherry picked from commit 9c214520b49d5dd7e7d7da2bb8dbbc8bcc26cdfe) --- base/fft/FFTW.jl | 10 +++++----- test/fft.jl | 3 +++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/base/fft/FFTW.jl b/base/fft/FFTW.jl index 509081c23b60d..fba05d2b3b3fa 100644 --- a/base/fft/FFTW.jl +++ b/base/fft/FFTW.jl @@ -459,7 +459,7 @@ for (Tr,Tc,fftw,lib) in ((:Float64,:Complex128,"fftw",libfftw), region, flags::Integer, timelimit::Real) direction = K set_timelimit($Tr, timelimit) - R = copy(region) + R = isa(region, Tuple) ? region : copy(region) dims, howmany = dims_howmany(X, Y, [size(X)...], R) plan = ccall(($(string(fftw,"_plan_guru64_dft")),$lib), PlanPtr, @@ -477,7 +477,7 @@ for (Tr,Tc,fftw,lib) in ((:Float64,:Complex128,"fftw",libfftw), @eval function (::Type{rFFTWPlan{$Tr,$FORWARD,inplace,N}}){inplace,N}(X::StridedArray{$Tr,N}, Y::StridedArray{$Tc,N}, region, flags::Integer, timelimit::Real) - R = copy(region) + R = isa(region, Tuple) ? region : copy(region) region = circshift([region...],-1) # FFTW halves last dim set_timelimit($Tr, timelimit) dims, howmany = dims_howmany(X, Y, [size(X)...], region) @@ -497,7 +497,7 @@ for (Tr,Tc,fftw,lib) in ((:Float64,:Complex128,"fftw",libfftw), @eval function (::Type{rFFTWPlan{$Tc,$BACKWARD,inplace,N}}){inplace,N}(X::StridedArray{$Tc,N}, Y::StridedArray{$Tr,N}, region, flags::Integer, timelimit::Real) - R = copy(region) + R = isa(region, Tuple) ? region : copy(region) region = circshift([region...],-1) # FFTW halves last dim set_timelimit($Tr, timelimit) dims, howmany = dims_howmany(X, Y, [size(Y)...], region) @@ -518,7 +518,7 @@ for (Tr,Tc,fftw,lib) in ((:Float64,:Complex128,"fftw",libfftw), Y::StridedArray{$Tr,N}, region, kinds, flags::Integer, timelimit::Real) - R = copy(region) + R = isa(region, Tuple) ? region : copy(region) knd = fix_kinds(region, kinds) set_timelimit($Tr, timelimit) dims, howmany = dims_howmany(X, Y, [size(X)...], region) @@ -540,7 +540,7 @@ for (Tr,Tc,fftw,lib) in ((:Float64,:Complex128,"fftw",libfftw), Y::StridedArray{$Tc,N}, region, kinds, flags::Integer, timelimit::Real) - R = copy(region) + R = isa(region, Tuple) ? region : copy(region) knd = fix_kinds(region, kinds) set_timelimit($Tr, timelimit) dims, howmany = dims_howmany(X, Y, [size(X)...], region) diff --git a/test/fft.jl b/test/fft.jl index 6aca96d874a02..6905bc85ffb30 100644 --- a/test/fft.jl +++ b/test/fft.jl @@ -3,6 +3,9 @@ # fft a = rand(8) + im*rand(8) @test norm(ifft(fft(a)) - a) < 1e-8 +@test norm(ifft(fft(a,1),1) - a) < 1e-8 +@test norm(ifft(fft(a,[1]),[1]) - a) < 1e-8 +@test norm(ifft(fft(a,(1,)),(1,)) - a) < 1e-8 m4 = [16. 2 3 13; 5 11 10 8; From 5b1edb5eaaa705447a2b5ca391511a2a50017c9d Mon Sep 17 00:00:00 2001 From: kshyatt Date: Sat, 6 Aug 2016 10:16:39 -0700 Subject: [PATCH 30/80] More trig docstring cleanout (cherry picked from commit c7fc048c8ccf40112d386b8f133663cfa7a444de) ref #17871 --- base/docs/helpdb/Base.jl | 42 ---------------------------------------- base/special/trig.jl | 9 ++++++--- 2 files changed, 6 insertions(+), 45 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 82604311cd1cc..463fac30e923e 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -976,13 +976,6 @@ seeding. """ srand -""" - acot(x) - -Compute the inverse cotangent of `x`, where the output is in radians. -""" -acot - """ oftype(x, y) @@ -2640,13 +2633,6 @@ Get the concrete type of `x`. """ typeof -""" - acsc(x) - -Compute the inverse cosecant of `x`, where the output is in radians -""" -acsc - """ log(x) @@ -3059,13 +3045,6 @@ Element-wise less-than-or-equals comparison operator. """ Base.:(.<=) -""" - asec(x) - -Compute the inverse secant of `x`, where the output is in radians. -""" -asec - """ rank(M) @@ -4925,13 +4904,6 @@ recurses infinitely. """ StackOverflowError -""" - acsch(x) - -Compute the inverse hyperbolic cosecant of `x`. -""" -acsch - """ process_running(p::Process) @@ -5689,13 +5661,6 @@ Read a UDP packet from the specified socket, and return the bytes received. This """ recv -""" - acoth(x) - -Compute the inverse hyperbolic cotangent of `x`. -""" -acoth - """ det(M) @@ -6038,13 +6003,6 @@ the integrity and correctness of data read from `stream`. """ deserialize -""" - asech(x) - -Compute the inverse hyperbolic secant of `x`. -""" -asech - """ ismarked(s) diff --git a/base/special/trig.jl b/base/special/trig.jl index 087f2ad0338e6..fa4c33717f784 100644 --- a/base/special/trig.jl +++ b/base/special/trig.jl @@ -319,10 +319,13 @@ for (finv, f) in ((:sec, :cos), (:csc, :sin), (:cot, :tan), end end -for (fa, fainv) in ((:asec, :acos), (:acsc, :asin), (:acot, :atan), - (:asech, :acosh), (:acsch, :asinh), (:acoth, :atanh)) +for (fa, fainv, fn) in ((:asec, :acos, "secant"), (:acsc, :asin, "cosecant"), (:acot, :atan, "cotangent"), + (:asech, :acosh, "hyperbolic secant"), (:acsch, :asinh, "hyperbolic cosecant"), (:acoth, :atanh, "hyperbolic cotangent")) + name = string(fa) @eval begin - ($fa){T<:Number}(y::T) = ($fainv)(one(T) / y) + @doc """ + $($name)(x) + Compute the $($fn) of `x`, where the output is in radians. """ ($fa){T<:Number}(y::T) = ($fainv)(one(T) / y) ($fa){T<:Number}(y::AbstractArray{T}) = ($fainv)(one(T) ./ y) end end From 91a3c42021911240589ed3b36ec936df2ccfd9c5 Mon Sep 17 00:00:00 2001 From: kshyatt Date: Sun, 7 Aug 2016 16:35:56 -0700 Subject: [PATCH 31/80] Remove dupe digamma, fix docstrings (cherry picked from commit 2912c9e2adb05258793d8b25931fa0b87720a4c5) ref #17871 --- base/docs/helpdb/Base.jl | 7 ------- base/special/trig.jl | 18 ++++++++++++------ doc/stdlib/math.rst | 2 +- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 463fac30e923e..e84547a23a682 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -9,13 +9,6 @@ Raises a `SystemError` for `errno` with the descriptive string `sysfunc` if `ift """ systemerror -""" - digamma(x) - -Compute the digamma function of `x` (the logarithmic derivative of `gamma(x)`) -""" -digamma - """ fill!(A, x) diff --git a/base/special/trig.jl b/base/special/trig.jl index fa4c33717f784..2ad94211c590f 100644 --- a/base/special/trig.jl +++ b/base/special/trig.jl @@ -319,14 +319,20 @@ for (finv, f) in ((:sec, :cos), (:csc, :sin), (:cot, :tan), end end -for (fa, fainv, fn) in ((:asec, :acos, "secant"), (:acsc, :asin, "cosecant"), (:acot, :atan, "cotangent"), - (:asech, :acosh, "hyperbolic secant"), (:acsch, :asinh, "hyperbolic cosecant"), (:acoth, :atanh, "hyperbolic cotangent")) - name = string(fa) +for (tfa, tfainv, hfa, hfainv, fn) in ((:asec, :acos, :asech, :acosh, "secant"), + (:acsc, :asin, :acsch, :asinh, "cosecant"), + (:acot, :atan, :acoth, :atahn, "cotangent")) + tname = string(tfa) + hname = string(hfa) @eval begin @doc """ - $($name)(x) - Compute the $($fn) of `x`, where the output is in radians. """ ($fa){T<:Number}(y::T) = ($fainv)(one(T) / y) - ($fa){T<:Number}(y::AbstractArray{T}) = ($fainv)(one(T) ./ y) + $($tname)(x) + Compute the inverse $($fn) of `x`, where the output is in radians. """ ($tfa){T<:Number}(y::T) = ($tfainv)(one(T) / y) + ($tfa){T<:Number}(y::AbstractArray{T}) = ($tfainv)(one(T) ./ y) + @doc """ + $($hname)(x) + Compute the inverse hyperbolic $($fn) of `x`. """ ($hfa){T<:Number}(y::T) = ($hfainv)(one(T) / y) + ($hfa){T<:Number}(y::AbstractArray{T}) = ($hfainv)(one(T) ./ y) end end diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index 7c9a7c653e808..91b6c7d926867 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -664,7 +664,7 @@ Mathematical Functions .. Docstring generated from Julia source - Compute the inverse cosecant of ``x``\ , where the output is in radians + Compute the inverse cosecant of ``x``\ , where the output is in radians. .. function:: acot(x) From 87565d3a889caa5a80a0ff29adfbbde9f30f78f2 Mon Sep 17 00:00:00 2001 From: Ranjan Anantharaman Date: Mon, 8 Aug 2016 16:46:22 +0530 Subject: [PATCH 32/80] Improve fft documentation (#17778) * Improve fft documentation * Remove whitespace * Use new notes syntax * Commit the generated .rst * Talk about cores, use CPU_CORES. * Committing generated rst (again) * `Other languages` -> `FFT libraries in other languages` * Remove redundant parts, add period * Link to section of manual * Fix doctests from changing line numbers * Fix doctests once more (cherry picked from commit 887ca0d7207f271a9620fa02a47eefb634158516) --- base/dft.jl | 12 ++++++++++-- doc/stdlib/math.rst | 9 +++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/base/dft.jl b/base/dft.jl index 6379968e022ea..04b9c5bf7875b 100644 --- a/base/dft.jl +++ b/base/dft.jl @@ -395,8 +395,16 @@ if Base.USE_GPL_LIBS A multidimensional FFT simply performs this operation along each transformed dimension of `A`. - Higher performance is usually possible with multi-threading. Use `FFTW.set_num_threads(np)` - to use `np` threads, if you have `np` processors. + !!! note + * Julia starts FFTW up with 1 thread by default. Higher performance is usually possible by + increasing number of threads. Use `FFTW.set_num_threads(Sys.CPU_CORES)` to use as many + threads as cores on your system. + + * This performs a multidimensional FFT by default. FFT libraries in other languages such as + Python and Octave perform a one-dimensional FFT along the first non-singleton dimension + of the array. This is worth noting while performing comparisons. For more details, + refer to the ["Noteworthy Differences from other Languages"](:ref:`man-noteworthy-differences`) + section of the manual. """ -> fft diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index 91b6c7d926867..cf4a4ad72312d 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -1805,9 +1805,7 @@ Signal Processing Fast Fourier transform (FFT) functions in Julia are implemented by calling functions from `FFTW -`_. By default, Julia does not use multi-threaded -FFTW. Higher performance may be obtained by experimenting with -multi-threading. Use ``FFTW.set_num_threads(np)`` to use ``np`` threads. +`_. .. function:: fft(A [, dims]) @@ -1826,7 +1824,10 @@ multi-threading. Use ``FFTW.set_num_threads(np)`` to use ``np`` threads. A multidimensional FFT simply performs this operation along each transformed dimension of ``A``\ . - Higher performance is usually possible with multi-threading. Use ``FFTW.set_num_threads(np)`` to use ``np`` threads, if you have ``np`` processors. + .. note:: + * Julia starts FFTW up with 1 thread by default. Higher performance is usually possible by increasing number of threads. Use ``FFTW.set_num_threads(Sys.CPU_CORES)`` to use as many threads as cores on your system. + * This performs a multidimensional FFT by default. FFT libraries in other languages such as Python and Octave perform a one-dimensional FFT along the first non-singleton dimension of the array. This is worth noting while performing comparisons. For more details, refer to the :ref:`man-noteworthy-differences` section of the manual. + .. function:: fft!(A [, dims]) From df978c3f45a922c01eb780691e81496cde331ed3 Mon Sep 17 00:00:00 2001 From: Christopher Rackauckas Date: Mon, 8 Aug 2016 09:56:16 -0700 Subject: [PATCH 33/80] Add Gitter to README (#17878) * Add Gitter to README The Gitter is a large active community that new users have found lots of help at. I think it would be helpful to have the Gitter right here, because some of the people went to IRC for chat only to find out about the Gitter through the Gitter/IRC bot (and then switched to Gtiter for the code highlighting). I think showing the active community and the help it gives on a README will be important to lower the new-user threshold. * Removed Google+, and moved Gitter above IRC (cherry picked from commit 2277270e637859a820c378d32c7fd583af209923) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6baaf4c2343a1..ff00afed1dd92 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ This is the GitHub repository of Julia source code, including instructions for c - **Source code:** - **Git clone URL:** - **Mailing lists:** +- **Gitter:** - **IRC:** - **Code coverage:** @@ -42,7 +43,6 @@ developers may find the notes in [CONTRIBUTING](https://github.com/JuliaLang/jul - [**StackOverflow**](https://stackoverflow.com/questions/tagged/julia-lang) - [**Youtube**](https://www.youtube.com/channel/UC9IuUwwE2xdjQUT_LMLONoA) - [**Twitter**](https://twitter.com/JuliaLanguage) -- [**Google+**](https://plus.google.com/communities/111295162646766639102) - [**Meetup**](http://julia.meetup.com/) From d09dafbf986552627eb0dbd045204dca9481879b Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Fri, 5 Aug 2016 13:57:05 -0400 Subject: [PATCH 34/80] sharper inference of `===` with singletons, e.g. `x===nothing` (cherry picked from commit 283bd843e17e3440935cf8e4ce960dce72ef9ee6) ref #17844 --- base/inference.jl | 3 +++ test/inference.jl | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/base/inference.jl b/base/inference.jl index 90ae3d82356d9..aa97ef4f710f5 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -286,6 +286,9 @@ add_tfunc(is, 2, 2, return Const(x.parameters[1]===y.parameters[1]) elseif typeintersect(widenconst(x), widenconst(y)) === Bottom return Const(false) + elseif (isa(x,Const) && y === typeof(x.val) && isdefined(y,:instance)) || + (isa(y,Const) && x === typeof(y.val) && isdefined(x,:instance)) + return Const(true) else return Bool end diff --git a/test/inference.jl b/test/inference.jl index 5776354b91e6e..dc8677df4d71e 100644 --- a/test/inference.jl +++ b/test/inference.jl @@ -269,3 +269,8 @@ function f17572{A}(::Type{Val{A}}) end # test that inference doesn't error @test isa(code_typed(f17572, (Type{Val{0}},)), Array) + +# === with singleton constants +let f(x) = (x===nothing) ? 1 : 1.0 + @test Base.return_types(f, (Void,)) == Any[Int] +end From 7b5782bddc993343b2ae2263ae5632fdc05b23fa Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sun, 7 Aug 2016 01:19:58 -0400 Subject: [PATCH 35/80] move the typecache and inference locks to protect more of their global state (cherry picked from commit 7017d932e59b0ebae271056d745444a47c1575ed) ref #17885 --- base/inference.jl | 6 ++++-- doc/devdocs/locks.rst | 6 ------ src/jltypes.c | 23 +++++++++++++++-------- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index aa97ef4f710f5..4022da0726a60 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -1487,6 +1487,7 @@ function typeinf_edge(method::Method, atypes::ANY, sparams::SimpleVector, needtr linfo = specialize_method(method, atypes, sparams, cached) end + ccall(:jl_typeinf_begin, Void, ()) # XXX: the following logic is likely subtly broken if code.code was nothing, # although it seems unlikely something bad (infinite recursion) will happen as a result if linfo.inInference @@ -1529,6 +1530,7 @@ function typeinf_edge(method::Method, atypes::ANY, sparams::SimpleVector, needtr end end typeinf_loop(frame) + ccall(:jl_typeinf_end, Void, ()) return (frame.linfo, widenconst(frame.bestguess), frame.inferred) end @@ -1576,8 +1578,10 @@ function typeinf_ext(linfo::LambdaInfo) else # toplevel lambda - infer directly linfo.inInference = true + ccall(:jl_typeinf_begin, Void, ()) frame = InferenceState(linfo, true, inlining_enabled(), true) typeinf_loop(frame) + ccall(:jl_typeinf_end, Void, ()) @assert frame.inferred # TODO: deal with this better return linfo end @@ -1591,7 +1595,6 @@ function typeinf_loop(frame) frame.inworkq || typeinf_frame(frame) return end - ccall(:jl_typeinf_begin, Void, ()) try in_typeinf_loop = true # the core type-inference algorithm @@ -1638,7 +1641,6 @@ function typeinf_loop(frame) println(ex) ccall(:jlbacktrace, Void, ()) end - ccall(:jl_typeinf_end, Void, ()) nothing end diff --git a/doc/devdocs/locks.rst b/doc/devdocs/locks.rst index 191312d811323..ea37560bd2eaf 100644 --- a/doc/devdocs/locks.rst +++ b/doc/devdocs/locks.rst @@ -118,12 +118,6 @@ The following locks are broken: fix: create it -* typecache - - this only protects cache lookup and insertion but it doesn't make the lookup-and-construct atomic - - fix: lock for ``apply_type`` / global (level 2?) - Shared Global Data Structures ----------------------------- diff --git a/src/jltypes.c b/src/jltypes.c index d4977719145b9..74f54e98eb768 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -1997,10 +1997,8 @@ static jl_value_t *lookup_type(jl_typename_t *tn, jl_value_t **key, size_t n) { JL_TIMING(TYPE_CACHE_LOOKUP); int ord = is_typekey_ordered(key, n); - JL_LOCK(&typecache_lock); // Might GC ssize_t idx = lookup_type_idx(tn, key, n, ord); jl_value_t *t = (idx < 0) ? NULL : jl_svecref(ord ? tn->cache : tn->linearcache, idx); - JL_UNLOCK(&typecache_lock); // Might GC return t; } @@ -2071,14 +2069,12 @@ jl_value_t *jl_cache_type_(jl_datatype_t *type) if (is_cacheable(type)) { JL_TIMING(TYPE_CACHE_INSERT); int ord = is_typekey_ordered(jl_svec_data(type->parameters), jl_svec_len(type->parameters)); - JL_LOCK(&typecache_lock); // Might GC ssize_t idx = lookup_type_idx(type->name, jl_svec_data(type->parameters), jl_svec_len(type->parameters), ord); if (idx >= 0) type = (jl_datatype_t*)jl_svecref(ord ? type->name->cache : type->name->linearcache, idx); else cache_insert_type((jl_value_t*)type, ~idx, ord); - JL_UNLOCK(&typecache_lock); // Might GC } return (jl_value_t*)type; } @@ -2163,13 +2159,18 @@ static jl_value_t *inst_datatype(jl_datatype_t *dt, jl_svec_t *p, jl_value_t **i int istuple = (tn == jl_tuple_typename); // check type cache if (cacheable) { + JL_LOCK(&typecache_lock); // Might GC jl_value_t *lkup = (jl_value_t*)lookup_type(tn, iparams, ntp); - if (lkup != NULL) + if (lkup != NULL) { + JL_UNLOCK(&typecache_lock); // Might GC return lkup; + } } jl_value_t *stack_lkup = lookup_type_stack(stack, dt, ntp, iparams); - if (stack_lkup) + if (stack_lkup) { + if (cacheable) JL_UNLOCK(&typecache_lock); // Might GC return stack_lkup; + } if (istuple && ntp > 0 && jl_is_vararg_type(iparams[ntp - 1])) { // normalize Tuple{..., Vararg{Int, 3}} to Tuple{..., Int, Int, Int} @@ -2180,6 +2181,7 @@ static jl_value_t *inst_datatype(jl_datatype_t *dt, jl_svec_t *p, jl_value_t **i jl_errorf("apply_type: Vararg length N is negative: %zd", nt); va = jl_tparam0(va); if (nt == 0 || !jl_has_typevars(va)) { + if (cacheable) JL_UNLOCK(&typecache_lock); // Might GC if (ntp == 1) return jl_tupletype_fill(nt, va); size_t i, l; @@ -2206,10 +2208,13 @@ static jl_value_t *inst_datatype(jl_datatype_t *dt, jl_svec_t *p, jl_value_t **i jl_type_error_rt("apply_type", "Vararg count", (jl_value_t*)jl_long_type, iparams[1]); } } - if (tc != (jl_value_t*)dt) + if (tc != (jl_value_t*)dt) { + if (cacheable) JL_UNLOCK(&typecache_lock); // Might GC return (jl_value_t*)jl_apply_type_(tc, iparams, ntp); + } } else if (ntp == 0 && jl_emptytuple != NULL) { + if (cacheable) JL_UNLOCK(&typecache_lock); // Might GC return jl_typeof(jl_emptytuple); } @@ -2299,8 +2304,10 @@ static jl_value_t *inst_datatype(jl_datatype_t *dt, jl_svec_t *p, jl_value_t **i else ndt->ninitialized = dt->ninitialized; - if (cacheable) + if (cacheable) { jl_cache_type_(ndt); + JL_UNLOCK(&typecache_lock); // Might GC + } JL_GC_POP(); return (jl_value_t*)ndt; From 96af21a548347c934f61e821ceb41687ecd8079d Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 4 Aug 2016 17:52:35 -0400 Subject: [PATCH 36/80] make Libuv types RAII with better error handling and bump libuv version for bugfixes (cherry picked from commit 2fdc182b79f906b7370e19c266f2119f1f58eb14) ref #17824 --- base/process.jl | 9 +- base/socket.jl | 115 +++++++-------- base/stream.jl | 131 ++++++++---------- .../md5 | 1 + .../sha512 | 1 + .../md5 | 1 - .../sha512 | 1 - deps/libuv.version | 2 +- test/spawn.jl | 17 +++ 9 files changed, 137 insertions(+), 141 deletions(-) create mode 100644 deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/md5 create mode 100644 deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/sha512 delete mode 100644 deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/md5 delete mode 100644 deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/sha512 diff --git a/base/process.jl b/base/process.jl index 88d51cb4f1634..22c39e4527f64 100644 --- a/base/process.jl +++ b/base/process.jl @@ -403,22 +403,19 @@ end function setup_stdio(stdio::PipeEndpoint, readable::Bool) closeafter = false - if stdio.handle == C_NULL - io = Libc.malloc(_sizeof_uv_named_pipe) + if stdio.status == StatusUninit if readable link_pipe(io, false, stdio, true) else link_pipe(stdio, true, io, false) end closeafter = true - else - io = stdio.handle end - return (io, closeafter) + return (stdio.handle, closeafter) end function setup_stdio(stdio::Pipe, readable::Bool) - if stdio.in.handle == C_NULL && stdio.out.handle == C_NULL + if stdio.in.status == StatusUninit && stdio.out.status == StatusUninit link_pipe(stdio) end io = readable ? stdio.out : stdio.in diff --git a/base/socket.jl b/base/socket.jl index 8365f219fe711..d037173ea62b5 100644 --- a/base/socket.jl +++ b/base/socket.jl @@ -255,33 +255,30 @@ type TCPSocket <: LibuvStream lock::ReentrantLock throttle::Int - TCPSocket(handle) = new( - handle, - StatusUninit, - true, - PipeBuffer(), - false, Condition(), - false, Condition(), - false, Condition(), - nothing, - ReentrantLock(), - DEFAULT_READ_BUFFER_SZ - ) + function TCPSocket(handle::Ptr{Void}, status) + tcp = new( + handle, + status, + true, + PipeBuffer(), + false, Condition(), + false, Condition(), + false, Condition(), + nothing, + ReentrantLock(), + DEFAULT_READ_BUFFER_SZ) + associate_julia_struct(tcp.handle, tcp) + finalizer(tcp, uvfinalize) + return tcp + end end function TCPSocket() - this = TCPSocket(Libc.malloc(_sizeof_uv_tcp)) - associate_julia_struct(this.handle,this) - finalizer(this,uvfinalize) - err = ccall(:uv_tcp_init,Cint,(Ptr{Void},Ptr{Void}), - eventloop(),this.handle) - if err != 0 - #TODO: this codepath is not currently tested - Libc.free(this.handle) - this.handle = C_NULL - throw(UVError("failed to create tcp socket",err)) - end - this.status = StatusInit - return this + tcp = TCPSocket(Libc.malloc(_sizeof_uv_tcp), StatusUninit) + err = ccall(:uv_tcp_init, Cint, (Ptr{Void}, Ptr{Void}), + eventloop(), tcp.handle) + uv_error("failed to create tcp socket", err) + tcp.status = StatusInit + return tcp end type TCPServer <: LibuvServer @@ -292,27 +289,24 @@ type TCPServer <: LibuvServer closecb::Callback closenotify::Condition - TCPServer(handle) = new( - handle, - StatusUninit, - false, Condition(), - false, Condition() - ) + function TCPServer(handle::Ptr{Void}, status) + tcp = new( + handle, + status, + false, Condition(), + false, Condition()) + associate_julia_struct(tcp.handle, tcp) + finalizer(tcp, uvfinalize) + return tcp + end end function TCPServer() - this = TCPServer(Libc.malloc(_sizeof_uv_tcp)) - associate_julia_struct(this.handle, this) - finalizer(this,uvfinalize) - err = ccall(:uv_tcp_init,Cint,(Ptr{Void},Ptr{Void}), - eventloop(),this.handle) - if err != 0 - #TODO: this codepath is not currently tested - Libc.free(this.handle) - this.handle = C_NULL - throw(UVError("failed to create tcp server",err)) - end - this.status = StatusInit - return this + tcp = TCPServer(Libc.malloc(_sizeof_uv_tcp), StatusUninit) + err = ccall(:uv_tcp_init, Cint, (Ptr{Void}, Ptr{Void}), + eventloop(), tcp.handle) + uv_error("failed to create tcp server", err) + tcp.status = StatusInit + return tcp end isreadable(io::TCPSocket) = isopen(io) || nb_available(io) > 0 @@ -344,26 +338,23 @@ type UDPSocket <: LibuvStream sendnotify::Condition closenotify::Condition - UDPSocket(handle::Ptr) = new( - handle, - StatusUninit, - Condition(), - Condition(), - Condition() - ) + function UDPSocket(handle::Ptr{Void}, status) + udp = new( + handle, + status, + Condition(), + Condition(), + Condition()) + associate_julia_struct(udp.handle, udp) + finalizer(udp, uvfinalize) + return udp + end end function UDPSocket() - this = UDPSocket(Libc.malloc(_sizeof_uv_udp)) - associate_julia_struct(this.handle, this) - err = ccall(:uv_udp_init,Cint,(Ptr{Void},Ptr{Void}), - eventloop(),this.handle) - finalizer(this, uvfinalize) - if err != 0 - #TODO: this codepath is not currently tested - Libc.free(this.handle) - this.handle = C_NULL - throw(UVError("failed to create udp socket",err)) - end + this = UDPSocket(Libc.malloc(_sizeof_uv_udp), StatusUninit) + err = ccall(:uv_udp_init, Cint, (Ptr{Void}, Ptr{Void}), + eventloop(), this.handle) + uv_error("failed to create udp socket", err) this.status = StatusInit return this end diff --git a/base/stream.jl b/base/stream.jl index 733bba8d450ce..2c7ad60244ccd 100644 --- a/base/stream.jl +++ b/base/stream.jl @@ -113,16 +113,21 @@ type PipeEndpoint <: LibuvStream lock::ReentrantLock throttle::Int - PipeEndpoint(handle::Ptr{Void} = C_NULL) = new( - handle, - StatusUninit, - PipeBuffer(), - true, - false,Condition(), - false,Condition(), - false,Condition(), - nothing, ReentrantLock(), - DEFAULT_READ_BUFFER_SZ) + PipeEndpoint() = PipeEndpoint(Libc.malloc(_sizeof_uv_named_pipe), StatusUninit) + function PipeEndpoint(handle::Ptr{Void}, status) + p = new(handle, + status, + PipeBuffer(), + true, + false,Condition(), + false,Condition(), + false,Condition(), + nothing, ReentrantLock(), + DEFAULT_READ_BUFFER_SZ) + associate_julia_struct(handle, p) + finalizer(p, uvfinalize) + return p + end end type PipeServer <: LibuvServer @@ -132,26 +137,22 @@ type PipeServer <: LibuvServer connectnotify::Condition closecb::Callback closenotify::Condition - PipeServer(handle) = new( - handle, - StatusUninit, - false,Condition(), - false,Condition()) + function PipeServer(handle::Ptr{Void}, status) + p = new(handle, + status, + false,Condition(), + false,Condition()) + associate_julia_struct(p.handle, p) + finalizer(p, uvfinalize) + return p + end end typealias LibuvPipe Union{PipeEndpoint, PipeServer} function PipeServer() - handle = Libc.malloc(_sizeof_uv_named_pipe) - try - ret = PipeServer(handle) - associate_julia_struct(ret.handle,ret) - finalizer(ret,uvfinalize) - return init_pipe!(ret;readable=true) - catch - Libc.free(handle) - rethrow() - end + p = PipeServer(Libc.malloc(_sizeof_uv_named_pipe), StatusUninit) + return init_pipe!(p; readable=true) end type TTY <: LibuvStream @@ -167,16 +168,19 @@ type TTY <: LibuvStream lock::ReentrantLock throttle::Int @static if is_windows(); ispty::Bool; end - function TTY(handle) + TTY() = TTY(Libc.malloc(_sizeof_uv_tty), StatusUninit) + function TTY(handle::Ptr{Void}, status) tty = new( handle, - StatusUninit, + status, true, PipeBuffer(), false,Condition(), false,Condition(), nothing, ReentrantLock(), DEFAULT_READ_BUFFER_SZ) + associate_julia_struct(handle, tty) + finalizer(tty, uvfinalize) @static if is_windows() tty.ispty = ccall(:jl_ispty, Cint, (Ptr{Void},), handle) != 0 end @@ -185,16 +189,15 @@ type TTY <: LibuvStream end function TTY(fd::RawFD; readable::Bool = false) - handle = Libc.malloc(_sizeof_uv_tty) - ret = TTY(handle) - associate_julia_struct(handle,ret) - finalizer(ret,uvfinalize) + tty = TTY() # This needs to go after associate_julia_struct so that there # is no garbage in the ->data field - uv_error("TTY",ccall(:uv_tty_init,Int32,(Ptr{Void},Ptr{Void},Int32,Int32),eventloop(),handle,fd.fd,readable)) - ret.status = StatusOpen - ret.line_buffered = false - return ret + err = ccall(:uv_tty_init, Int32, (Ptr{Void}, Ptr{Void}, Int32, Int32), + eventloop(), tty.handle, fd.fd, readable) + uv_error("TTY", err) + tty.status = StatusOpen + tty.line_buffered = false + return tty end show(io::IO,stream::LibuvServer) = print(io, typeof(stream), "(", uv_status_string(stream), ")") @@ -230,18 +233,15 @@ function init_stdio(handle::Ptr{Void}) # return File(RawFD(ccall(:jl_uv_file_handle,Int32,(Ptr{Void},),handle))) else if t == UV_TTY - ret = TTY(handle) + ret = TTY(handle, StatusOpen) elseif t == UV_TCP - ret = TCPSocket(handle) + ret = TCPSocket(handle, StatusOpen) elseif t == UV_NAMED_PIPE - ret = PipeEndpoint(handle) + ret = PipeEndpoint(handle, StatusOpen) else throw(ArgumentError("invalid stdio type: $t")) end - ret.status = StatusOpen ret.line_buffered = false - associate_julia_struct(ret.handle, ret) - finalizer(ret, uvfinalize) return ret end end @@ -322,7 +322,9 @@ function wait_close(x::Union{LibuvStream, LibuvServer}) end function close(stream::Union{LibuvStream, LibuvServer}) - if isopen(stream) + if stream.status == StatusInit + ccall(:jl_forceclose_uv, Void, (Ptr{Void},), stream.handle) + elseif isopen(stream) if stream.status != StatusClosing ccall(:jl_close_uv, Void, (Ptr{Void},), stream.handle) stream.status = StatusClosing @@ -337,11 +339,13 @@ end function uvfinalize(uv::Union{LibuvStream, LibuvServer}) if uv.handle != C_NULL disassociate_julia_struct(uv.handle) # not going to call the usual close hooks - if uv.status != StatusUninit && uv.status != StatusInit + if uv.status != StatusUninit close(uv) - uv.handle = C_NULL - uv.status = StatusClosed + else + Libc.free(uv.handle) end + uv.status = StatusClosed + uv.handle = C_NULL end nothing end @@ -571,24 +575,23 @@ function init_pipe!(pipe::LibuvPipe; if pipe.status != StatusUninit error("pipe is already initialized") end - if pipe.handle == C_NULL - malloc_julia_pipe!(pipe) - end - uv_error("init_pipe",ccall(:jl_init_pipe, Cint, + err = ccall(:jl_init_pipe, Cint, (Ptr{Void}, Int32, Int32, Int32), - pipe.handle, writable, readable, julia_only)) + pipe.handle, writable, readable, julia_only) + uv_error( + if readable && writable + "init_pipe(ipc)" + elseif readable + "init_pipe(read)" + elseif writable + "init_pipe(write)" + else + "init_pipe(none)" + end, err) pipe.status = StatusInit return pipe end -function malloc_julia_pipe!(x::LibuvPipe) - assert(x.handle == C_NULL) - x.handle = Libc.malloc(_sizeof_uv_named_pipe) - associate_julia_struct(x.handle, x) - finalizer(x, uvfinalize) - nothing -end - function _link_pipe(read_end::Ptr{Void}, write_end::Ptr{Void}) uv_error("pipe_link", ccall(:uv_pipe_link, Int32, (Ptr{Void}, Ptr{Void}), read_end, write_end)) @@ -620,9 +623,6 @@ end function link_pipe(read_end::PipeEndpoint, readable_julia_only::Bool, write_end::Ptr{Void}, writable_julia_only::Bool) - if read_end.handle == C_NULL - malloc_julia_pipe!(read_end) - end init_pipe!(read_end; readable = true, writable = false, julia_only = readable_julia_only) uv_error("init_pipe", @@ -634,9 +634,6 @@ end function link_pipe(read_end::Ptr{Void}, readable_julia_only::Bool, write_end::PipeEndpoint, writable_julia_only::Bool) - if write_end.handle == C_NULL - malloc_julia_pipe!(write_end) - end uv_error("init_pipe", ccall(:jl_init_pipe, Cint, (Ptr{Void},Int32,Int32,Int32), read_end, 0, 1, readable_julia_only)) init_pipe!(write_end; @@ -648,12 +645,6 @@ end function link_pipe(read_end::PipeEndpoint, readable_julia_only::Bool, write_end::PipeEndpoint, writable_julia_only::Bool) - if write_end.handle == C_NULL - malloc_julia_pipe!(write_end) - end - if read_end.handle == C_NULL - malloc_julia_pipe!(read_end) - end init_pipe!(read_end; readable = true, writable = false, julia_only = readable_julia_only) init_pipe!(write_end; diff --git a/deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/md5 b/deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/md5 new file mode 100644 index 0000000000000..e42a20ffd5a35 --- /dev/null +++ b/deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/md5 @@ -0,0 +1 @@ +cc07a8ef026fa42eeaddf7b6e074096e diff --git a/deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/sha512 b/deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/sha512 new file mode 100644 index 0000000000000..bec2d8d44203f --- /dev/null +++ b/deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/sha512 @@ -0,0 +1 @@ +e128cec9548ff2f52a78743203d3f06dceef3cf7cbeb3a7a5f7453b132576833f82688eed52cf74c035f57828deac079cd845ad47c8cd6197a8e0bebf3586fc2 diff --git a/deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/md5 b/deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/md5 deleted file mode 100644 index 793f153e26212..0000000000000 --- a/deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -c6a019d79d20eabc39619a04961c9a3b diff --git a/deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/sha512 b/deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/sha512 deleted file mode 100644 index f3f1ec9dca483..0000000000000 --- a/deps/checksums/libuv-cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -478ab473244b01bef344892a75e09fef50da8fb1a7212e0257c53f3223de4fde5f6bd449eef34bc1f025481c7d9f854002acb6eb203b447a50a34bae4ad9dee4 diff --git a/deps/libuv.version b/deps/libuv.version index 51f79989c5fc3..478ebca66589a 100644 --- a/deps/libuv.version +++ b/deps/libuv.version @@ -1,2 +1,2 @@ LIBUV_BRANCH=julia-uv1.9.0 -LIBUV_SHA1=cb6d0f875a5b8ca30cba45c0c1ef7442c87c1e68 +LIBUV_SHA1=28743d6091531340cfe316de2b2d385fe1778ff5 diff --git a/test/spawn.jl b/test/spawn.jl index 7fa04587e3b28..b4d7f7ee1990e 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -392,3 +392,20 @@ end @test_throws ArgumentError reduce(&, Base.AbstractCmd[]) @test_throws ArgumentError reduce(&, Base.Cmd[]) @test reduce(&, [`$echo abc`, `$echo def`, `$echo hij`]) == `$echo abc` & `$echo def` & `$echo hij` + +# test for proper handling of FD exhaustion +let ps = Pipe[] + try + for i = 1:100_000 + p = Pipe() + Base.link_pipe(p) + push!(ps, p) + end + @test false + catch ex + for p in ps + close(p) + end + @test (ex::Base.UVError).code == Base.UV_EMFILE + end +end From 0a8b9a24b7901b7a7a4e6e7ccf58518682ad5a5e Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 6 Aug 2016 12:54:57 -0400 Subject: [PATCH 37/80] only run Pipe exhaustion test on unix windows doesn't stop you from allocating millions of pipes, until Julia runs out of memory (cherry picked from commit 3cfd1e418b65e5dfb41c40b7a7b4016b814a43d6) ref #17824 --- test/spawn.jl | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/test/spawn.jl b/test/spawn.jl index b4d7f7ee1990e..cfff6b6b6425a 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -394,18 +394,20 @@ end @test reduce(&, [`$echo abc`, `$echo def`, `$echo hij`]) == `$echo abc` & `$echo def` & `$echo hij` # test for proper handling of FD exhaustion -let ps = Pipe[] - try - for i = 1:100_000 - p = Pipe() - Base.link_pipe(p) - push!(ps, p) - end - @test false - catch ex - for p in ps - close(p) +if is_unix() + let ps = Pipe[] + try + for i = 1:100_000 + p = Pipe() + Base.link_pipe(p) + push!(ps, p) + end + @test false + catch ex + for p in ps + close(p) + end + @test (ex::Base.UVError).code == Base.UV_EMFILE end - @test (ex::Base.UVError).code == Base.UV_EMFILE end end From 3d9c3d180f79f40f7c0aff951e0f5c75776a15f8 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 4 May 2016 18:02:37 -0400 Subject: [PATCH 38/80] improve efficiency of Dict and accuracy of age (nee dirty) flag ref comment in #15923 (cherry picked from commit 32b58e1061a1fa8aa3080ba4a2a1bd394d13b34c) ref #16204 --- base/dict.jl | 106 ++++++++++++++++++++------------------------------- 1 file changed, 42 insertions(+), 64 deletions(-) diff --git a/base/dict.jl b/base/dict.jl index 00e56434d2745..ae7635bf1cdb6 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -331,13 +331,13 @@ type Dict{K,V} <: Associative{K,V} vals::Array{V,1} ndel::Int count::Int - dirty::Bool + age::UInt idxfloor::Int # an index <= the indexes of all used slots maxprobe::Int function Dict() n = 16 - new(zeros(UInt8,n), Array{K}(n), Array{V}(n), 0, 0, false, 1, 0) + new(zeros(UInt8,n), Array{K,1}(n), Array{V,1}(n), 0, 0, 0, 1, 0) end function Dict(kv) h = Dict{K,V}() @@ -360,7 +360,7 @@ type Dict{K,V} <: Associative{K,V} rehash!(d) end @assert d.ndel == 0 - new(copy(d.slots), copy(d.keys), copy(d.vals), 0, d.count, d.dirty, d.idxfloor, + new(copy(d.slots), copy(d.keys), copy(d.vals), 0, d.count, d.age, d.idxfloor, d.maxprobe) end end @@ -427,7 +427,7 @@ function convert{K,V}(::Type{Dict{K,V}},d::Associative) end convert{K,V}(::Type{Dict{K,V}},d::Dict{K,V}) = d -hashindex(key, sz) = ((hash(key)%Int) & (sz-1)) + 1 +hashindex(key, sz) = (((hash(key)%Int) & (sz-1)) + 1)::Int isslotempty(h::Dict, i::Int) = h.slots[i] == 0x0 isslotfilled(h::Dict, i::Int) = h.slots[i] == 0x1 @@ -439,7 +439,7 @@ function rehash!{K,V}(h::Dict{K,V}, newsz = length(h.keys)) oldv = h.vals sz = length(olds) newsz = _tablesz(newsz) - h.dirty = true + h.age += 1 h.idxfloor = 1 if h.count == 0 resize!(h.slots, newsz) @@ -451,9 +451,9 @@ function rehash!{K,V}(h::Dict{K,V}, newsz = length(h.keys)) end slots = zeros(UInt8,newsz) - keys = Array{K}(newsz) - vals = Array{V}(newsz) - count0 = h.count + keys = Array{K,1}(newsz) + vals = Array{V,1}(newsz) + age0 = h.age count = 0 maxprobe = h.maxprobe @@ -472,8 +472,8 @@ function rehash!{K,V}(h::Dict{K,V}, newsz = length(h.keys)) vals[index] = v count += 1 - if h.count != count0 - # if items are removed by finalizers, retry + if h.age != age0 + # if `h` is changed by a finalizer, retry return rehash!(h, newsz) end end @@ -485,6 +485,7 @@ function rehash!{K,V}(h::Dict{K,V}, newsz = length(h.keys)) h.count = count h.ndel = 0 h.maxprobe = maxprobe + @assert h.age == age0 return h end @@ -511,7 +512,7 @@ function empty!{K,V}(h::Dict{K,V}) resize!(h.vals, sz) h.ndel = 0 h.count = 0 - h.dirty = true + h.age += 1 h.idxfloor = 1 return h end @@ -528,7 +529,7 @@ function ht_keyindex{K,V}(h::Dict{K,V}, key) if isslotempty(h,index) break end - if !isslotmissing(h,index) && isequal(key,keys[index]) + if !isslotmissing(h,index) && (key === keys[index] || isequal(key,keys[index])) return index end @@ -543,6 +544,7 @@ end # and the key would be inserted at pos # This version is for use by setindex! and get! function ht_keyindex2{K,V}(h::Dict{K,V}, key) + age0 = h.age sz = length(h.keys) iter = 0 maxprobe = h.maxprobe @@ -552,7 +554,9 @@ function ht_keyindex2{K,V}(h::Dict{K,V}, key) while true if isslotempty(h,index) - avail < 0 && return avail + if avail < 0 + return avail + end return -index end @@ -562,7 +566,7 @@ function ht_keyindex2{K,V}(h::Dict{K,V}, key) # in case "key" already exists in a later collided slot. avail = -index end - elseif isequal(key, keys[index]) + elseif key === keys[index] || isequal(key, keys[index]) return index end @@ -594,7 +598,7 @@ function _setindex!(h::Dict, v, key, index) h.keys[index] = key h.vals[index] = v h.count += 1 - h.dirty = true + h.age += 1 if index < h.idxfloor h.idxfloor = index end @@ -620,6 +624,7 @@ function setindex!{K,V}(h::Dict{K,V}, v0, key::K) index = ht_keyindex2(h, key) if index > 0 + h.age += 1 h.keys[index] = key h.vals[index] = v else @@ -629,30 +634,13 @@ function setindex!{K,V}(h::Dict{K,V}, v0, key::K) return h end -function get!{K,V}(h::Dict{K,V}, key0, default) - key = convert(K,key0) - if !isequal(key,key0) - throw(ArgumentError("$key0 is not a valid key for type $K")) - end - get!(h, key, default) -end - -function get!{K, V}(h::Dict{K,V}, key::K, default) - index = ht_keyindex2(h, key) - - index > 0 && return h.vals[index] - - v = convert(V, default) - _setindex!(h, v, key, -index) - return v -end - +get!{K,V}(h::Dict{K,V}, key0, default) = get!(()->default, h, key0) function get!{K,V}(default::Callable, h::Dict{K,V}, key0) - key = convert(K,key0) - if !isequal(key,key0) + key = convert(K, key0) + if !isequal(key, key0) throw(ArgumentError("$key0 is not a valid key for type $K")) end - get!(default, h, key) + return get!(default, h, key) end function get!{K,V}(default::Callable, h::Dict{K,V}, key::K) @@ -660,12 +648,13 @@ function get!{K,V}(default::Callable, h::Dict{K,V}, key::K) index > 0 && return h.vals[index] - h.dirty = false - v = convert(V, default()) - if h.dirty + age0 = h.age + v = convert(V, default()) + if h.age != age0 index = ht_keyindex2(h, key) end if index > 0 + h.age += 1 h.keys[index] = key h.vals[index] = v else @@ -674,41 +663,28 @@ function get!{K,V}(default::Callable, h::Dict{K,V}, key::K) return v end -# NOTE: this macro is specific to Dict, not Associative, and should +# NOTE: this macro is trivial, and should # therefore not be exported as-is: it's for internal use only. macro get!(h, key0, default) - quote - K, V = keytype($(esc(h))), valtype($(esc(h))) - key = convert(K, $(esc(key0))) - if !isequal(key, $(esc(key0))) - throw(ArgumentError(string($(esc(key0)), " is not a valid key for type ", K))) - end - idx = ht_keyindex2($(esc(h)), key) - if idx < 0 - idx = -idx - v = convert(V, $(esc(default))) - _setindex!($(esc(h)), v, key, idx) - else - @inbounds v = $(esc(h)).vals[idx] - end - v + return quote + get!(()->$(esc(default)), $(esc(h)), $(esc(key0))) end end function getindex{K,V}(h::Dict{K,V}, key) index = ht_keyindex(h, key) - return (index<0) ? throw(KeyError(key)) : h.vals[index]::V + return (index < 0) ? throw(KeyError(key)) : h.vals[index]::V end function get{K,V}(h::Dict{K,V}, key, default) index = ht_keyindex(h, key) - return (index<0) ? default : h.vals[index]::V + return (index < 0) ? default : h.vals[index]::V end function get{K,V}(default::Callable, h::Dict{K,V}, key) index = ht_keyindex(h, key) - return (index<0) ? default() : h.vals[index]::V + return (index < 0) ? default() : h.vals[index]::V end haskey(h::Dict, key) = (ht_keyindex(h, key) >= 0) @@ -727,12 +703,12 @@ end function pop!(h::Dict, key) index = ht_keyindex(h, key) - index > 0 ? _pop!(h, index) : throw(KeyError(key)) + return index > 0 ? _pop!(h, index) : throw(KeyError(key)) end function pop!(h::Dict, key, default) index = ht_keyindex(h, key) - index > 0 ? _pop!(h, index) : default + return index > 0 ? _pop!(h, index) : default end function _delete!(h::Dict, index) @@ -741,14 +717,16 @@ function _delete!(h::Dict, index) ccall(:jl_arrayunset, Void, (Any, UInt), h.vals, index-1) h.ndel += 1 h.count -= 1 - h.dirty = true - h + h.age += 1 + return h end function delete!(h::Dict, key) index = ht_keyindex(h, key) - if index > 0; _delete!(h, index); end - h + if index > 0 + _delete!(h, index) + end + return h end function skip_deleted(h::Dict, i) From 25c23b721573ac0a1c26dd2d782c4ecf8f3f6b02 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 25 Jul 2016 17:26:52 -0400 Subject: [PATCH 39/80] split weakkeydict into a separate file (cherry picked from commit 609957fcebe6b7d1a695d0e27807acdb0d4bfd4a) ref #16204 --- base/dict.jl | 63 +---------------------------------------- base/sysimg.jl | 1 + base/weakkeydict.jl | 69 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 62 deletions(-) create mode 100644 base/weakkeydict.jl diff --git a/base/dict.jl b/base/dict.jl index ae7635bf1cdb6..e0f947cd23a1a 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -751,70 +751,9 @@ length(t::Dict) = t.count next{T<:Dict}(v::KeyIterator{T}, i) = (v.dict.keys[i], skip_deleted(v.dict,i+1)) next{T<:Dict}(v::ValueIterator{T}, i) = (v.dict.vals[i], skip_deleted(v.dict,i+1)) -# weak key dictionaries - -type WeakKeyDict{K,V} <: Associative{K,V} - ht::Dict{Any,V} - deleter::Function - - WeakKeyDict() = new(Dict{Any,V}(), identity) -end -WeakKeyDict() = WeakKeyDict{Any,Any}() - -function weak_key_delete!(t::Dict, k) - # when a weak key is finalized, remove from dictionary if it is still there - wk = getkey(t, k, secret_table_token) - if !is(wk,secret_table_token) && is(wk.value, k) - delete!(t, k) - end -end - -function setindex!{K}(wkh::WeakKeyDict{K}, v, key) - t = wkh.ht - k = convert(K, key) - if is(wkh.deleter, identity) - wkh.deleter = x->weak_key_delete!(t, x) - end - t[WeakRef(k)] = v - # TODO: it might be better to avoid the finalizer, allow - # wiped WeakRefs to remain in the table, and delete them as - # they are discovered by getindex and setindex!. - finalizer(k, wkh.deleter) - return t -end - - -function getkey{K}(wkh::WeakKeyDict{K}, kk, default) - k = getkey(wkh.ht, kk, secret_table_token) - if is(k, secret_table_token) - return default - end - return k.value::K -end - -get{K}(wkh::WeakKeyDict{K}, key, default) = get(wkh.ht, key, default) -get{K}(default::Callable, wkh::WeakKeyDict{K}, key) = get(default, wkh.ht, key) -get!{K}(wkh::WeakKeyDict{K}, key, default) = get!(wkh.ht, key, default) -get!{K}(default::Callable, wkh::WeakKeyDict{K}, key) = get!(default, wkh.ht, key) -pop!{K}(wkh::WeakKeyDict{K}, key) = pop!(wkh.ht, key) -pop!{K}(wkh::WeakKeyDict{K}, key, default) = pop!(wkh.ht, key, default) -delete!{K}(wkh::WeakKeyDict{K}, key) = delete!(wkh.ht, key) -empty!(wkh::WeakKeyDict) = (empty!(wkh.ht); wkh) -haskey{K}(wkh::WeakKeyDict{K}, key) = haskey(wkh.ht, key) -getindex{K}(wkh::WeakKeyDict{K}, key) = getindex(wkh.ht, key) -isempty(wkh::WeakKeyDict) = isempty(wkh.ht) - -start(t::WeakKeyDict) = start(t.ht) -done(t::WeakKeyDict, i) = done(t.ht, i) -function next{K,V}(t::WeakKeyDict{K,V}, i) - kv, i = next(t.ht, i) - (Pair{K,V}(kv[1].value::K,kv[2]), i) -end -length(t::WeakKeyDict) = length(t.ht) - # For these Associative types, it is safe to implement filter! # by deleting keys during iteration. -function filter!(f, d::Union{ObjectIdDict,Dict,WeakKeyDict}) +function filter!(f, d::Union{ObjectIdDict,Dict}) for (k,v) in d if !f(k,v) delete!(d,k) diff --git a/base/sysimg.jl b/base/sysimg.jl index f2cf7fb13d96d..9461866639070 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -115,6 +115,7 @@ include("reshapedarray.jl") include("bitarray.jl") include("intset.jl") include("dict.jl") +include("weakkeydict.jl") include("set.jl") include("iterator.jl") diff --git a/base/weakkeydict.jl b/base/weakkeydict.jl new file mode 100644 index 0000000000000..a1f5a0ded821e --- /dev/null +++ b/base/weakkeydict.jl @@ -0,0 +1,69 @@ +# weak key dictionaries + +type WeakKeyDict{K,V} <: Associative{K,V} + ht::Dict{Any,V} + deleter::Function + + WeakKeyDict() = new(Dict{Any,V}(), identity) +end +WeakKeyDict() = WeakKeyDict{Any,Any}() + +function weak_key_delete!(t::Dict, k) + # when a weak key is finalized, remove from dictionary if it is still there + wk = getkey(t, k, secret_table_token) + if !is(wk,secret_table_token) && is(wk.value, k) + delete!(t, k) + end +end + +function setindex!{K}(wkh::WeakKeyDict{K}, v, key) + t = wkh.ht + k = convert(K, key) + if is(wkh.deleter, identity) + wkh.deleter = x->weak_key_delete!(t, x) + end + t[WeakRef(k)] = v + # TODO: it might be better to avoid the finalizer, allow + # wiped WeakRefs to remain in the table, and delete them as + # they are discovered by getindex and setindex!. + finalizer(k, wkh.deleter) + return t +end + + +function getkey{K}(wkh::WeakKeyDict{K}, kk, default) + k = getkey(wkh.ht, kk, secret_table_token) + if is(k, secret_table_token) + return default + end + return k.value::K +end + +get{K}(wkh::WeakKeyDict{K}, key, default) = get(wkh.ht, key, default) +get{K}(default::Callable, wkh::WeakKeyDict{K}, key) = get(default, wkh.ht, key) +get!{K}(wkh::WeakKeyDict{K}, key, default) = get!(wkh.ht, key, default) +get!{K}(default::Callable, wkh::WeakKeyDict{K}, key) = get!(default, wkh.ht, key) +pop!{K}(wkh::WeakKeyDict{K}, key) = pop!(wkh.ht, key) +pop!{K}(wkh::WeakKeyDict{K}, key, default) = pop!(wkh.ht, key, default) +delete!{K}(wkh::WeakKeyDict{K}, key) = delete!(wkh.ht, key) +empty!(wkh::WeakKeyDict) = (empty!(wkh.ht); wkh) +haskey{K}(wkh::WeakKeyDict{K}, key) = haskey(wkh.ht, key) +getindex{K}(wkh::WeakKeyDict{K}, key) = getindex(wkh.ht, key) +isempty(wkh::WeakKeyDict) = isempty(wkh.ht) + +start(t::WeakKeyDict) = start(t.ht) +done(t::WeakKeyDict, i) = done(t.ht, i) +function next{K,V}(t::WeakKeyDict{K,V}, i) + kv, i = next(t.ht, i) + (Pair{K,V}(kv[1].value::K,kv[2]), i) +end +length(t::WeakKeyDict) = length(t.ht) + +function filter!(f, d::WeakKeyDict) + for (k, v) in d + if !f(k, v) + delete!(d, k) + end + end + return d +end From 7db1cd9bec043f9c21ab273901daa0b6c6337dbd Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 25 Jul 2016 17:32:15 -0400 Subject: [PATCH 40/80] move Thread support much earlier in bootstrapping (cherry picked from commit bbd4bcc5db90a4ad73898f6f8113a18342ff1ea3) ref #16204 --- base/sysimg.jl | 16 ++++++++-------- base/threads.jl | 1 - 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/base/sysimg.jl b/base/sysimg.jl index 9461866639070..8428a4a5166ec 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -154,6 +154,12 @@ include("show.jl") include("base64.jl") importall .Base64 +# nullable types +include("nullable.jl") + +# version +include("version.jl") + # system & environment include("libc.jl") using .Libc: getpid, gethostname, time @@ -161,14 +167,12 @@ include("libdl.jl") using .Libdl: DL_LOAD_PATH include("env.jl") -# nullable types -include("nullable.jl") - # Scheduling include("libuv.jl") include("event.jl") include("task.jl") include("lock.jl") +include("threads.jl") # I/O include("stream.jl") @@ -218,9 +222,6 @@ include("collections.jl") include("sort.jl") importall .Sort -# version -include("version.jl") - function deepcopy_internal end # BigInts and BigFloats @@ -343,8 +344,7 @@ import .Dates: Date, DateTime, now include("sparse/sparse.jl") importall .SparseArrays -# threads -include("threads.jl") +# worker threads include("threadcall.jl") # deprecated functions diff --git a/base/threads.jl b/base/threads.jl index 4550e41dd0ba7..ef9f6957fa3b7 100644 --- a/base/threads.jl +++ b/base/threads.jl @@ -7,4 +7,3 @@ include("atomics.jl") include("locks.jl") end - From b6737487d626c2f2465eb35c55adf001c73b158f Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 25 Jul 2016 19:41:50 -0400 Subject: [PATCH 41/80] make WeakKeyDict finalizer usage gc-safe also use this `client_refs.lock` to protect other data-structures from being interrupted by finalizers, in the multi.jl logic we may want to start indicating which mutable data-structures are safe to call from finalizers, since generally that isn't possible to make a finalizer API gc-safe, that code should observe the standard thread-safe restrictions (there's no guarantee of which thread it'll run on), plus, if the data-structures uses locks for synchronization, use the `islocked` pattern (demonstrated herein) in the `finalizer` to re-schedule the finalizer when the mutable data-structure is not available for mutation. this ensures that the lock cannot be acquired recursively, and furthermore, this pattern will continue to work if finalizers get moved to their own separate thread. close #14445 fix #16550 reverts workaround #14456 (shouldn't break #14295, due to new locks) should fix #16091 (with #17619) (cherry picked from commit cd8be658c1cc954a28b538aa7200c62b739da936) ref #16204 --- base/dict.jl | 10 --- base/lock.jl | 20 ++++++ base/multi.jl | 155 ++++++++++++++++++++++++-------------------- base/precompile.jl | 1 - base/serialize.jl | 11 +++- base/sysimg.jl | 2 +- base/weakkeydict.jl | 88 ++++++++++++++----------- test/misc.jl | 17 ++++- 8 files changed, 180 insertions(+), 124 deletions(-) diff --git a/base/dict.jl b/base/dict.jl index e0f947cd23a1a..8990bb198c1a2 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -309,16 +309,6 @@ get!(o::ObjectIdDict, key, default) = (o[key] = get(o, key, default)) abstract AbstractSerializer -# Serializer type needed as soon as ObjectIdDict is available -type SerializationState{I<:IO} <: AbstractSerializer - io::I - counter::Int - table::ObjectIdDict - SerializationState(io::I) = new(io, 0, ObjectIdDict()) -end - -SerializationState(io::IO) = SerializationState{typeof(io)}(io) - # dict # These can be changed, to trade off better performance for space diff --git a/base/lock.jl b/base/lock.jl index ea42b8e454dd2..e0bd86dfeff81 100644 --- a/base/lock.jl +++ b/base/lock.jl @@ -95,6 +95,26 @@ function unlock(rl::ReentrantLock) return end +function lock(f, l) + lock(l) + try + return f() + finally + unlock(l) + end +end + +function trylock(f, l) + if trylock(l) + try + return f() + finally + unlock(l) + end + end + return false +end + """ Semaphore(sem_size) diff --git a/base/multi.jl b/base/multi.jl index e299edb35bda3..3049e5732fd2e 100644 --- a/base/multi.jl +++ b/base/multi.jl @@ -637,27 +637,38 @@ function deregister_worker(pg, pid) # delete this worker from our remote reference client sets ids = [] tonotify = [] - for (id,rv) in pg.refs - if in(pid,rv.clientset) - push!(ids, id) + lock(client_refs) do + for (id,rv) in pg.refs + if in(pid,rv.clientset) + push!(ids, id) + end + if rv.waitingfor == pid + push!(tonotify, (id,rv)) + end end - if rv.waitingfor == pid - push!(tonotify, (id,rv)) + for id in ids + del_client(pg, id, pid) end - end - for id in ids - del_client(pg, id, pid) - end - # throw exception to tasks waiting for this pid - for (id,rv) in tonotify - notify_error(rv.c, ProcessExitedException()) - delete!(pg.refs, id) + # throw exception to tasks waiting for this pid + for (id,rv) in tonotify + notify_error(rv.c, ProcessExitedException()) + delete!(pg.refs, id) + end end end ## remote refs ## -const client_refs = WeakKeyDict() + +""" + client_refs + +Tracks whether a particular AbstractRemoteRef +(identified by its RRID) exists on this worker. + +The client_refs lock is also used to synchronize access to `.refs` and associated clientset state +""" +const client_refs = WeakKeyDict{Any, Void}() # used as a WeakKeySet abstract AbstractRemoteRef @@ -680,34 +691,26 @@ type RemoteChannel{T<:AbstractChannel} <: AbstractRemoteRef end function test_existing_ref(r::AbstractRemoteRef) - found = getkey(client_refs, r, false) - if found !== false - if client_refs[r] == true - @assert r.where > 0 - if isa(r, Future) && isnull(found.v) && !isnull(r.v) - # we have recd the value from another source, probably a deserialized ref, send a del_client message - send_del_client(r) - found.v = r.v - end - return found - else - # just delete the entry. - delete!(client_refs, found) - end + found = getkey(client_refs, r, nothing) + if found !== nothing + @assert r.where > 0 + if isa(r, Future) && isnull(found.v) && !isnull(r.v) + # we have recd the value from another source, probably a deserialized ref, send a del_client message + send_del_client(r) + found.v = r.v + end + return found::typeof(r) end - client_refs[r] = true + client_refs[r] = nothing finalizer(r, finalize_ref) return r end function finalize_ref(r::AbstractRemoteRef) - if r.where > 0 # Handle the case of the finalizer having being called manually - if haskey(client_refs, r) - # NOTE: Change below line to deleting the entry once issue https://github.com/JuliaLang/julia/issues/14445 - # is fixed. - client_refs[r] = false - end + if r.where > 0 # Handle the case of the finalizer having been called manually + islocked(client_refs) && return finalizer(r, finalize_ref) # delay finalizer for later, when it's not already locked + delete!(client_refs, r) if isa(r, RemoteChannel) send_del_client(r) else @@ -717,7 +720,7 @@ function finalize_ref(r::AbstractRemoteRef) end r.where = 0 end - return r + nothing end Future(w::LocalProcess) = Future(w.id) @@ -791,23 +794,27 @@ A low-level API which returns the backing `AbstractChannel` for an `id` returned The call is valid only on the node where the backing channel exists. """ function channel_from_id(id) - rv = get(PGRP.refs, id, false) + rv = lock(client_refs) do + return get(PGRP.refs, id, false) + end if rv === false throw(ErrorException("Local instance of remote reference not found")) end - rv.c + return rv.c end lookup_ref(rrid::RRID, f=def_rv_channel) = lookup_ref(PGRP, rrid, f) function lookup_ref(pg, rrid, f) - rv = get(pg.refs, rrid, false) - if rv === false - # first we've heard of this ref - rv = RemoteValue(f()) - pg.refs[rrid] = rv - push!(rv.clientset, rrid.whence) - end - rv + return lock(client_refs) do + rv = get(pg.refs, rrid, false) + if rv === false + # first we've heard of this ref + rv = RemoteValue(f()) + pg.refs[rrid] = rv + push!(rv.clientset, rrid.whence) + end + return rv + end::RemoteValue end """ @@ -827,7 +834,7 @@ function isready(rr::Future) !isnull(rr.v) && return true rid = remoteref_id(rr) - if rr.where == myid() + return if rr.where == myid() isready(lookup_ref(rid).c) else remotecall_fetch(rid->isready(lookup_ref(rid).c), rr.where, rid) @@ -844,7 +851,7 @@ it can be safely used on a `Future` since they are assigned only once. """ function isready(rr::RemoteChannel, args...) rid = remoteref_id(rr) - if rr.where == myid() + return if rr.where == myid() isready(lookup_ref(rid).c, args...) else remotecall_fetch(rid->isready(lookup_ref(rid).c, args...), rr.where, rid) @@ -855,11 +862,7 @@ del_client(rr::AbstractRemoteRef) = del_client(remoteref_id(rr), myid()) del_client(id, client) = del_client(PGRP, id, client) function del_client(pg, id, client) -# As a workaround to issue https://github.com/JuliaLang/julia/issues/14445 -# the dict/set updates are executed asynchronously so that they do -# not occur in the midst of a gc. The `@async` prefix must be removed once -# 14445 is fixed. - @async begin + lock(client_refs) do rv = get(pg.refs, id, false) if rv !== false delete!(rv.clientset, client) @@ -898,8 +901,10 @@ function send_del_client(rr) end function add_client(id, client) - rv = lookup_ref(id) - push!(rv.clientset, client) + lock(client_refs) do + rv = lookup_ref(id) + push!(rv.clientset, client) + end nothing end @@ -999,7 +1004,7 @@ function run_work_thunk(thunk, print_error) result = RemoteException(ce) print_error && showerror(STDERR, ce) end - result + return result end function run_work_thunk(rv::RemoteValue, thunk) put!(rv, run_work_thunk(thunk, false)) @@ -1007,11 +1012,13 @@ function run_work_thunk(rv::RemoteValue, thunk) end function schedule_call(rid, thunk) - rv = RemoteValue(def_rv_channel()) - (PGRP::ProcessGroup).refs[rid] = rv - push!(rv.clientset, rid.whence) - schedule(@task(run_work_thunk(rv,thunk))) - rv + return lock(client_refs) do + rv = RemoteValue(def_rv_channel()) + (PGRP::ProcessGroup).refs[rid] = rv + push!(rv.clientset, rid.whence) + @schedule run_work_thunk(rv, thunk) + return rv + end end # make a thunk to call f on args in a way that simulates what would happen if @@ -1026,13 +1033,13 @@ end function remotecall(f, w::LocalProcess, args...; kwargs...) rr = Future(w) schedule_call(remoteref_id(rr), local_remotecall_thunk(f, args, kwargs)) - rr + return rr end function remotecall(f, w::Worker, args...; kwargs...) rr = Future(w) send_msg(w, MsgHeader(remoteref_id(rr)), CallMsg{:call}(f, args, kwargs)) - rr + return rr end """ @@ -1046,7 +1053,7 @@ remotecall(f, id::Integer, args...; kwargs...) = remotecall(f, worker_from_id(id function remotecall_fetch(f, w::LocalProcess, args...; kwargs...) v=run_work_thunk(local_remotecall_thunk(f,args, kwargs), false) - isa(v, RemoteException) ? throw(v) : v + return isa(v, RemoteException) ? throw(v) : v end function remotecall_fetch(f, w::Worker, args...; kwargs...) @@ -1057,8 +1064,10 @@ function remotecall_fetch(f, w::Worker, args...; kwargs...) rv.waitingfor = w.id send_msg(w, MsgHeader(RRID(0,0), oid), CallMsg{:call_fetch}(f, args, kwargs)) v = take!(rv) - delete!(PGRP.refs, oid) - isa(v, RemoteException) ? throw(v) : v + lock(client_refs) do + delete!(PGRP.refs, oid) + end + return isa(v, RemoteException) ? throw(v) : v end """ @@ -1080,9 +1089,11 @@ function remotecall_wait(f, w::Worker, args...; kwargs...) rr = Future(w) send_msg(w, MsgHeader(remoteref_id(rr), prid), CallWaitMsg(f, args, kwargs)) v = fetch(rv.c) - delete!(PGRP.refs, prid) + lock(client_refs) do + delete!(PGRP.refs, prid) + end isa(v, RemoteException) && throw(v) - rr + return rr end """ @@ -1834,9 +1845,11 @@ function create_worker(manager, wconfig) @schedule manage(w.manager, w.id, w.config, :register) wait(rr_ntfy_join) - delete!(PGRP.refs, ntfy_oid) + lock(client_refs) do + delete!(PGRP.refs, ntfy_oid) + end - w.id + return w.id end @@ -1859,7 +1872,7 @@ function launch_additional(np::Integer, cmd::Cmd) additional_io_objs[port] = io end - addresses + return addresses end function redirect_output_from_additional_worker(pid, port) diff --git a/base/precompile.jl b/base/precompile.jl index 9c8519886a1de..e189959ff2759 100644 --- a/base/precompile.jl +++ b/base/precompile.jl @@ -444,7 +444,6 @@ precompile(Base.get, (Base.Dict{Any, Any}, Tuple{Int64, Int64}, Bool)) precompile(Base.LineEdit.refresh_multi_line, (Array{Any, 1}, Base.Terminals.TerminalBuffer, Base.Terminals.TTYTerminal, Base.IOBuffer, Base.LineEdit.InputAreaState, Base.LineEdit.PromptState)) precompile(Base.schedule, (Array{Any, 1}, Task, Void)) precompile(Base.LineEdit.match_input, (Function, Base.LineEdit.MIState, Base.Terminals.TTYTerminal, Array{Char, 1}, Base.Dict{Char, Any})) -precompile(Base.weak_key_delete!, (Base.Dict{Any, Any}, Base.RemoteChannel)) precompile(==, (Base.RemoteChannel, WeakRef)) precompile(==, (Base.RemoteChannel, Base.RemoteChannel)) precompile(Base.send_del_client, (Base.RemoteChannel,)) diff --git a/base/serialize.jl b/base/serialize.jl index 55e5f0cdd868a..b50fe29937fc5 100644 --- a/base/serialize.jl +++ b/base/serialize.jl @@ -6,7 +6,16 @@ import Base: GMP, Bottom, unsafe_convert, uncompressed_ast, datatype_pointerfree import Core: svec using Base: ViewIndex, index_lengths -export serialize, deserialize +export serialize, deserialize, SerializationState + +type SerializationState{I<:IO} <: AbstractSerializer + io::I + counter::Int + table::ObjectIdDict + SerializationState(io::I) = new(io, 0, ObjectIdDict()) +end + +SerializationState(io::IO) = SerializationState{typeof(io)}(io) ## serializing values ## diff --git a/base/sysimg.jl b/base/sysimg.jl index 8428a4a5166ec..7005898b07050 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -115,7 +115,6 @@ include("reshapedarray.jl") include("bitarray.jl") include("intset.jl") include("dict.jl") -include("weakkeydict.jl") include("set.jl") include("iterator.jl") @@ -173,6 +172,7 @@ include("event.jl") include("task.jl") include("lock.jl") include("threads.jl") +include("weakkeydict.jl") # I/O include("stream.jl") diff --git a/base/weakkeydict.jl b/base/weakkeydict.jl index a1f5a0ded821e..6485c83af38e4 100644 --- a/base/weakkeydict.jl +++ b/base/weakkeydict.jl @@ -2,62 +2,74 @@ type WeakKeyDict{K,V} <: Associative{K,V} ht::Dict{Any,V} - deleter::Function + lock::Threads.RecursiveSpinLock + finalizer::Function - WeakKeyDict() = new(Dict{Any,V}(), identity) + function WeakKeyDict() + t = new(Dict{Any,V}(), Threads.RecursiveSpinLock(), identity) + t.finalizer = function(k) + # when a weak key is finalized, remove from dictionary if it is still there + islocked(t) && return finalizer(k, t.finalizer) + delete!(t, k) + end + return t + end end WeakKeyDict() = WeakKeyDict{Any,Any}() -function weak_key_delete!(t::Dict, k) - # when a weak key is finalized, remove from dictionary if it is still there - wk = getkey(t, k, secret_table_token) - if !is(wk,secret_table_token) && is(wk.value, k) - delete!(t, k) - end -end +islocked(wkh::WeakKeyDict) = islocked(wkh.lock) +lock(f, wkh::WeakKeyDict) = lock(f, wkh.lock) +trylock(f, wkh::WeakKeyDict) = trylock(f, wkh.lock) function setindex!{K}(wkh::WeakKeyDict{K}, v, key) - t = wkh.ht k = convert(K, key) - if is(wkh.deleter, identity) - wkh.deleter = x->weak_key_delete!(t, x) + finalizer(k, wkh.finalizer) + lock(wkh) do + wkh.ht[WeakRef(k)] = v end - t[WeakRef(k)] = v - # TODO: it might be better to avoid the finalizer, allow - # wiped WeakRefs to remain in the table, and delete them as - # they are discovered by getindex and setindex!. - finalizer(k, wkh.deleter) - return t + return wkh end - function getkey{K}(wkh::WeakKeyDict{K}, kk, default) - k = getkey(wkh.ht, kk, secret_table_token) - if is(k, secret_table_token) - return default + return lock(wkh) do + k = getkey(wkh.ht, kk, secret_table_token) + is(k, secret_table_token) && return default + return k.value::K end - return k.value::K end -get{K}(wkh::WeakKeyDict{K}, key, default) = get(wkh.ht, key, default) -get{K}(default::Callable, wkh::WeakKeyDict{K}, key) = get(default, wkh.ht, key) -get!{K}(wkh::WeakKeyDict{K}, key, default) = get!(wkh.ht, key, default) -get!{K}(default::Callable, wkh::WeakKeyDict{K}, key) = get!(default, wkh.ht, key) -pop!{K}(wkh::WeakKeyDict{K}, key) = pop!(wkh.ht, key) -pop!{K}(wkh::WeakKeyDict{K}, key, default) = pop!(wkh.ht, key, default) -delete!{K}(wkh::WeakKeyDict{K}, key) = delete!(wkh.ht, key) -empty!(wkh::WeakKeyDict) = (empty!(wkh.ht); wkh) -haskey{K}(wkh::WeakKeyDict{K}, key) = haskey(wkh.ht, key) -getindex{K}(wkh::WeakKeyDict{K}, key) = getindex(wkh.ht, key) +get{K}(wkh::WeakKeyDict{K}, key, default) = lock(() -> get(wkh.ht, key, default), wkh) +get{K}(default::Callable, wkh::WeakKeyDict{K}, key) = lock(() -> get(default, wkh.ht, key), wkh) +get!{K}(wkh::WeakKeyDict{K}, key, default) = lock(() -> get!(wkh.ht, key, default), wkh) +get!{K}(default::Callable, wkh::WeakKeyDict{K}, key) = lock(() -> get!(default, wkh.ht, key), wkh) +pop!{K}(wkh::WeakKeyDict{K}, key) = lock(() -> pop!(wkh.ht, key), wkh) +pop!{K}(wkh::WeakKeyDict{K}, key, default) = lock(() -> pop!(wkh.ht, key, default), wkh) +delete!{K}(wkh::WeakKeyDict{K}, key) = lock(() -> delete!(wkh.ht, key), wkh) +empty!(wkh::WeakKeyDict) = (lock(() -> empty!(wkh.ht)); wkh) +haskey{K}(wkh::WeakKeyDict{K}, key) = lock(() -> haskey(wkh.ht, key), wkh) +getindex{K}(wkh::WeakKeyDict{K}, key) = lock(() -> getindex(wkh.ht, key), wkh) isempty(wkh::WeakKeyDict) = isempty(wkh.ht) +length(t::WeakKeyDict) = length(t.ht) -start(t::WeakKeyDict) = start(t.ht) -done(t::WeakKeyDict, i) = done(t.ht, i) +function start{K,V}(t::WeakKeyDict{K,V}) + gc_token = Ref{Bool}(false) # no keys will be deleted via finalizers until this token is gc'd + finalizer(gc_token, function(r) + if r[] + r[] = false + unlock(t.lock) + end + end) + s = lock(t.lock) + gc_token[] = true + return (start(t.ht), gc_token) +end +done(t::WeakKeyDict, i) = done(t.ht, i[1]) function next{K,V}(t::WeakKeyDict{K,V}, i) - kv, i = next(t.ht, i) - (Pair{K,V}(kv[1].value::K,kv[2]), i) + gc_token = i[2] + wkv, i = next(t.ht, i[1]) + kv = Pair{K,V}(wkv[1].value::K, wkv[2]) + return (kv, (i, gc_token)) end -length(t::WeakKeyDict) = length(t.ht) function filter!(f, d::WeakKeyDict) for (k, v) in d diff --git a/test/misc.jl b/test/misc.jl index 633576319e108..d10b46aafa732 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -138,8 +138,21 @@ end # lock / unlock let l = ReentrantLock() lock(l) - @test trylock(l) - unlock(l) + success = Ref(false) + @test trylock(l) do + @test lock(l) do + success[] = true + return :foo + end === :foo + return :bar + end === :bar + @test success[] + t = @async begin + @test trylock(l) do + @test false + end === false + end + wait(t) unlock(l) @test_throws ErrorException unlock(l) end From d5a51e983e2e54e4d50fe768eb4c2224d4b67635 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Mon, 8 Aug 2016 11:53:57 -0500 Subject: [PATCH 42/80] Support -I for I::CartesianIndex (cherry picked from commit f6d3566550ab3975f4729876d3c9eed1fac86454) ref #17895 --- base/multidimensional.jl | 1 + test/arrayops.jl | 1 + 2 files changed, 2 insertions(+) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 385e9e218f891..272b528f964d9 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -48,6 +48,7 @@ one{N}(::CartesianIndex{N}) = one(CartesianIndex{N}) one{N}(::Type{CartesianIndex{N}}) = CartesianIndex(ntuple(x -> 1, Val{N})) # arithmetic, min/max +(-){N}(index::CartesianIndex{N}) = CartesianIndex{N}(map(-, index.I)) (+){N}(index1::CartesianIndex{N}, index2::CartesianIndex{N}) = CartesianIndex{N}(map(+, index1.I, index2.I)) (-){N}(index1::CartesianIndex{N}, index2::CartesianIndex{N}) = CartesianIndex{N}(map(-, index1.I, index2.I)) min{N}(index1::CartesianIndex{N}, index2::CartesianIndex{N}) = CartesianIndex{N}(map(min, index1.I, index2.I)) diff --git a/test/arrayops.jl b/test/arrayops.jl index bd66b597e1566..bacc66f668f13 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -1252,6 +1252,7 @@ end I1 = CartesianIndex((2,3,0)) I2 = CartesianIndex((-1,5,2)) +@test -I1 == CartesianIndex((-2,-3,0)) @test I1 + I2 == CartesianIndex((1,8,2)) @test I2 + I1 == CartesianIndex((1,8,2)) @test I1 - I2 == CartesianIndex((3,-2,-2)) From c0936b721e6c19888edfde4839cfc588a09b34b7 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Mon, 8 Aug 2016 11:54:17 -0500 Subject: [PATCH 43/80] Fix PermutedDimsArrays typo (cherry picked from commit 6ed3bb8fa848afb888bcc17aa89e7348a36b05ba) ref #17895 --- base/permuteddimsarray.jl | 2 +- test/arrayops.jl | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/base/permuteddimsarray.jl b/base/permuteddimsarray.jl index aa2944f7c4bb0..f940bd87e864b 100644 --- a/base/permuteddimsarray.jl +++ b/base/permuteddimsarray.jl @@ -17,7 +17,7 @@ immutable PermutedDimsArray{T,N,perm,iperm,AA<:AbstractArray} <: AbstractArray{T end function PermutedDimsArray{T,N}(data::AbstractArray{T,N}, perm) - length(perm) == N || throw(ArgumentError(string(p, " is not a valid permutation of dimensions 1:", N))) + length(perm) == N || throw(ArgumentError(string(perm, " is not a valid permutation of dimensions 1:", N))) iperm = invperm(perm) PermutedDimsArray{T,N,(perm...,),(iperm...,),typeof(data)}(data) end diff --git a/test/arrayops.jl b/test/arrayops.jl index bacc66f668f13..c33f17c05057c 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -446,7 +446,12 @@ s = view(a,:,[1,2,4],[1,5]) c = convert(Array, s) for p in ([1,2,3], [1,3,2], [2,1,3], [2,3,1], [3,1,2], [3,2,1]) @test permutedims(s, p) == permutedims(c, p) + @test Base.PermutedDimsArrays.PermutedDimsArray(s, p) == permutedims(c, p) end +@test_throws ArgumentError permutedims(a, (1,1,1)) +@test_throws ArgumentError permutedims(s, (1,1,1)) +@test_throws ArgumentError Base.PermutedDimsArrays.PermutedDimsArray(a, (1,1,1)) +@test_throws ArgumentError Base.PermutedDimsArrays.PermutedDimsArray(s, (1,1,1)) ## ipermutedims ## From 66b8bfedf05d068c5d81e3adf6002c970c7ecee9 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Mon, 8 Aug 2016 15:51:54 -0500 Subject: [PATCH 44/80] Add more backtrace tests (cherry picked from commit af2612d54405faf6bb50c36c52cb1d529c6d4f4b) ref #17818 --- test/backtrace.jl | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/test/backtrace.jl b/test/backtrace.jl index 8267c689c7635..bdc93b000e0d9 100644 --- a/test/backtrace.jl +++ b/test/backtrace.jl @@ -96,3 +96,46 @@ let @test i1 > 0 && i2 > 0 @test b1[i1].line != b2[i2].line end + +module BackTraceTesting + +using Base.Test + +@inline bt2() = backtrace() +@inline bt1() = bt2() +bt() = bt1() + +lkup = map(StackTraces.lookup, bt()) +hasbt = hasbt2 = false +for sfs in lkup + for sf in sfs + if sf.func == :bt + hasbt = true + end + if sf.func == :bt2 + hasbt2 = true + end + end +end +@test hasbt +@test_broken hasbt2 + +function btmacro() + @time backtrace() +end +lkup = map(StackTraces.lookup, btmacro()) +hasme = hasbtmacro = false +for sfs in lkup + for sf in sfs + if sf.func == Symbol("macro expansion") + hasme = true + end + if sf.func == :btmacro + hasbtmacro = true + end + end +end +@test hasme +@test hasbtmacro + +end From 8a1cf9b88ae178a250efd9afb1beedd5943881e7 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Mon, 8 Aug 2016 05:09:24 -0500 Subject: [PATCH 45/80] Faster, indices-aware circshift (and non-allocating circshift!) Fixes #16032, fixes #17581 (cherry picked from commit 60660b5dc663ee8c6f1501cc96c847ba64b14f5c) ref #17861 --- base/abstractarraymath.jl | 22 ++++++++--------- base/exports.jl | 1 + base/multidimensional.jl | 51 +++++++++++++++++++++++++++++++++++++++ doc/stdlib/arrays.rst | 16 ++++++++++-- test/arrayops.jl | 9 +++++++ test/offsetarray.jl | 1 + 6 files changed, 86 insertions(+), 14 deletions(-) diff --git a/base/abstractarraymath.jl b/base/abstractarraymath.jl index 7f95b1f1651ea..8abe1b19d86f9 100644 --- a/base/abstractarraymath.jl +++ b/base/abstractarraymath.jl @@ -158,8 +158,10 @@ function flipdim(A::AbstractArray, d::Integer) return B end -circshift(a::AbstractArray, shiftamt::Real) = circshift(a, [Integer(shiftamt)]) - +function circshift(a::AbstractArray, shiftamt::Real) + circshift!(similar(a), a, (Integer(shiftamt),)) +end +circshift(a::AbstractArray, shiftamt::DimsInteger) = circshift!(similar(a), a, shiftamt) """ circshift(A, shifts) @@ -174,29 +176,25 @@ julia> b = reshape(collect(1:16), (4,4)) 3 7 11 15 4 8 12 16 -julia> circshift(b, [0,2]) +julia> circshift(b, (0,2)) 4×4 Array{Int64,2}: 9 13 1 5 10 14 2 6 11 15 3 7 12 16 4 8 -julia> circshift(b, [-1,0]) +julia> circshift(b, (-1,0)) 4×4 Array{Int64,2}: 2 6 10 14 3 7 11 15 4 8 12 16 1 5 9 13 ``` + +See also `circshift!`. """ -function circshift{T,N}(a::AbstractArray{T,N}, shiftamts) - I = () - for i=1:N - s = size(a,i) - d = i<=length(shiftamts) ? shiftamts[i] : 0 - I = tuple(I..., d==0 ? [1:s;] : mod([-d:s-1-d;], s).+1) - end - a[(I::NTuple{N,Vector{Int}})...] +function circshift(a::AbstractArray, shiftamt) + circshift!(similar(a), a, map(Integer, (shiftamt...,))) end # Uses K-B-N summation diff --git a/base/exports.jl b/base/exports.jl index b92dfe12af6b9..33fe8c6850852 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -490,6 +490,7 @@ export checkbounds, checkindex, circshift, + circshift!, clamp!, colon, conj!, diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 272b528f964d9..a6e22e481601b 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -610,6 +610,57 @@ function copy!{T,N}(dest::AbstractArray{T,N}, src::AbstractArray{T,N}) dest end +function copy!(dest::AbstractArray, Rdest::CartesianRange, src::AbstractArray, Rsrc::CartesianRange) + isempty(Rdest) && return dest + size(Rdest) == size(Rsrc) || throw(ArgumentError("source and destination must have same size (got $(size(Rsrc)) and $(size(Rdest)))")) + @boundscheck checkbounds(dest, Rdest.start) + @boundscheck checkbounds(dest, Rdest.stop) + @boundscheck checkbounds(src, Rsrc.start) + @boundscheck checkbounds(src, Rsrc.stop) + deltaI = Rdest.start - Rsrc.start + for I in Rsrc + @inbounds dest[I+deltaI] = src[I] + end + dest +end + +# circshift! +circshift!(dest::AbstractArray, src, ::Tuple{}) = copy!(dest, src) +""" + circshift!(dest, src, shifts) + +Circularly shift the data in `src`, storing the result in +`dest`. `shifts` specifies the amount to shift in each dimension. + +The `dest` array must be distinct from the `src` array (they cannot +alias each other). + +See also `circshift`. +""" +@noinline function circshift!{T,N}(dest::AbstractArray{T,N}, src, shiftamt::DimsInteger) + dest === src && throw(ArgumentError("dest and src must be separate arrays")) + inds = indices(src) + indices(dest) == inds || throw(ArgumentError("indices of src and dest must match (got $inds and $(indices(dest)))")) + _circshift!(dest, (), src, (), inds, fill_to_length(shiftamt, 0, Val{N})) +end +circshift!(dest::AbstractArray, src, shiftamt) = circshift!(dest, src, (shiftamt...,)) + +@inline function _circshift!(dest, rdest, src, rsrc, + inds::Tuple{AbstractUnitRange,Vararg{Any}}, + shiftamt::Tuple{Integer,Vararg{Any}}) + ind1, d = inds[1], shiftamt[1] + s = mod(d, length(ind1)) + sf, sl = first(ind1)+s, last(ind1)-s + r1, r2 = first(ind1):sf-1, sf:last(ind1) + r3, r4 = first(ind1):sl, sl+1:last(ind1) + tinds, tshiftamt = tail(inds), tail(shiftamt) + _circshift!(dest, (rdest..., r1), src, (rsrc..., r4), tinds, tshiftamt) + _circshift!(dest, (rdest..., r2), src, (rsrc..., r3), tinds, tshiftamt) +end +# At least one of inds, shiftamt is empty +function _circshift!(dest, rdest, src, rsrc, inds, shiftamt) + copy!(dest, CartesianRange(rdest), src, CartesianRange(rsrc)) +end ### BitArrays diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index a475fd69b31ad..116433c9f3c5b 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -590,20 +590,32 @@ Indexing, Assignment, and Concatenation 3 7 11 15 4 8 12 16 - julia> circshift(b, [0,2]) + julia> circshift(b, (0,2)) 4×4 Array{Int64,2}: 9 13 1 5 10 14 2 6 11 15 3 7 12 16 4 8 - julia> circshift(b, [-1,0]) + julia> circshift(b, (-1,0)) 4×4 Array{Int64,2}: 2 6 10 14 3 7 11 15 4 8 12 16 1 5 9 13 + See also ``circshift!``\ . + +.. function:: circshift!(dest, src, shifts) + + .. Docstring generated from Julia source + + Circularly shift the data in ``src``\ , storing the result in ``dest``\ . ``shifts`` specifies the amount to shift in each dimension. + + The ``dest`` array must be distinct from the ``src`` array (they cannot alias each other). + + See also ``circshift``\ . + .. function:: find(A) .. Docstring generated from Julia source diff --git a/test/arrayops.jl b/test/arrayops.jl index c33f17c05057c..984128a58816a 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -462,6 +462,15 @@ for i = tensors @test isequal(i,permutedims(ipermutedims(i,perm),perm)) end +## circshift + +@test circshift(1:5, -1) == circshift(1:5, 4) == circshift(1:5, -6) == [2,3,4,5,1] +@test circshift(1:5, 1) == circshift(1:5, -4) == circshift(1:5, 6) == [5,1,2,3,4] +a = [1:5;] +@test_throws ArgumentError Base.circshift!(a, a, 1) +b = copy(a) +@test Base.circshift!(b, a, 1) == [5,1,2,3,4] + ## unique across dim ## # All rows and columns unique diff --git a/test/offsetarray.jl b/test/offsetarray.jl index 5ca538539ec6a..a509e17d3a8fb 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -411,6 +411,7 @@ v = OffsetArray(rand(8), (-2,)) @test rotr90(A) == OffsetArray(rotr90(parent(A)), A.offsets[[2,1]]) @test flipdim(A, 1) == OffsetArray(flipdim(parent(A), 1), A.offsets) @test flipdim(A, 2) == OffsetArray(flipdim(parent(A), 2), A.offsets) +@test circshift(A, (-1,2)) == OffsetArray(circshift(parent(A), (-1,2)), A.offsets) @test A+1 == OffsetArray(parent(A)+1, A.offsets) @test 2*A == OffsetArray(2*parent(A), A.offsets) From d02d911dad6f818fd5dc0440a950e384ef20349c Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Wed, 10 Aug 2016 21:43:21 -0700 Subject: [PATCH 46/80] Remove export and rst docs for circshift! on release-0.5 ref https://github.com/JuliaLang/julia/pull/17861#issuecomment-238505848 --- base/abstractarraymath.jl | 2 -- base/exports.jl | 1 - doc/stdlib/arrays.rst | 12 ------------ 3 files changed, 15 deletions(-) diff --git a/base/abstractarraymath.jl b/base/abstractarraymath.jl index 8abe1b19d86f9..2ffb6baf71130 100644 --- a/base/abstractarraymath.jl +++ b/base/abstractarraymath.jl @@ -190,8 +190,6 @@ julia> circshift(b, (-1,0)) 4 8 12 16 1 5 9 13 ``` - -See also `circshift!`. """ function circshift(a::AbstractArray, shiftamt) circshift!(similar(a), a, map(Integer, (shiftamt...,))) diff --git a/base/exports.jl b/base/exports.jl index 33fe8c6850852..b92dfe12af6b9 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -490,7 +490,6 @@ export checkbounds, checkindex, circshift, - circshift!, clamp!, colon, conj!, diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index 116433c9f3c5b..78749eac4ed17 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -604,18 +604,6 @@ Indexing, Assignment, and Concatenation 4 8 12 16 1 5 9 13 - See also ``circshift!``\ . - -.. function:: circshift!(dest, src, shifts) - - .. Docstring generated from Julia source - - Circularly shift the data in ``src``\ , storing the result in ``dest``\ . ``shifts`` specifies the amount to shift in each dimension. - - The ``dest`` array must be distinct from the ``src`` array (they cannot alias each other). - - See also ``circshift``\ . - .. function:: find(A) .. Docstring generated from Julia source From 4adaeee81cb31d56d0b2bbc3417802f0eb41ef3f Mon Sep 17 00:00:00 2001 From: Ranjan Anantharaman Date: Tue, 9 Aug 2016 20:22:32 +0530 Subject: [PATCH 47/80] Add test for @code_typed and @code_lowered (#17892) Add test for typed and lowered Check if @code_typed and @code_lowered return LambdaInfo instead of Array (cherry picked from commit f5eea9228ee05367571d79aea5c0b622d97d1569) --- test/reflection.jl | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/reflection.jl b/test/reflection.jl index 352ff8db8d9ea..bd832f25326a2 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -532,3 +532,23 @@ if is_windows() else @test isnull(h16850) end + +# Adds test for PR #17636 +let + a = @code_typed 1 + 1 + b = @code_lowered 1 + 1 + @test isa(a, LambdaInfo) + @test isa(b, LambdaInfo) + + function thing(a::Array, b::Real) + println("thing") + end + function thing(a::AbstractArray, b::Int) + println("blah") + end + @test_throws MethodError thing(rand(10), 1) + a = @code_typed thing(rand(10), 1) + b = @code_lowered thing(rand(10), 1) + @test length(a) == 0 + @test length(b) == 0 +end From 43d9f068073ef4723e1f08d16a33ca6c69aa53dc Mon Sep 17 00:00:00 2001 From: Ranjan Anantharaman Date: Tue, 9 Aug 2016 20:45:32 +0530 Subject: [PATCH 48/80] Add test for function name being assigned in function (#17835) For issue #4914 (cherry picked from commit 932ce0c261d069f003ad7c1820b5e9a0ca3abda8) --- test/core.jl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/core.jl b/test/core.jl index 394f7882dd0b3..ee4605ff60704 100644 --- a/test/core.jl +++ b/test/core.jl @@ -4461,3 +4461,11 @@ end #end #@test local_innersig(Int32(2)) == ((Int32(2), Int32(1), Int32(2)im), (Int32(2), UInt32(1))) #@test local_innersig(Int64(3)) == ((Int64(3), Int64(1), Int64(3)im), (Int64(3), UInt64(1))) + +# Issue 4914 +let + j(j) = j + @test j(1) == 1 + k(x) = (k = x; k) + @test k(1) == 1 +end From 9e896c9f68367e834e32eeb632da2b86a4e93775 Mon Sep 17 00:00:00 2001 From: Amit Murthy Date: Tue, 9 Aug 2016 21:46:20 +0530 Subject: [PATCH 49/80] Document `@threadcall` (#17915) (cherry picked from commit 42def4a1aec2a6402309619dda68e5948cfc4cb0) --- base/docs/helpdb/Base.jl | 2 +- doc/manual/parallel-computing.rst | 25 +++++++++++++++++++++++++ doc/stdlib/parallel.rst | 13 ++++++++++++- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index e84547a23a682..2175bd10b0a16 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -4839,7 +4839,7 @@ IntSet """ Task(func) -Create a `Task` (i.e. thread, or coroutine) to execute the given function (which must be +Create a `Task` (i.e. coroutine) to execute the given function (which must be callable with no arguments). The task exits when this function returns. """ Task diff --git a/doc/manual/parallel-computing.rst b/doc/manual/parallel-computing.rst index ca78dd0f6c4f6..9f9548ddf356b 100644 --- a/doc/manual/parallel-computing.rst +++ b/doc/manual/parallel-computing.rst @@ -1076,6 +1076,31 @@ The iteration space is split amongst the threads, after which each thread writes Note that :obj:`Threads.@threads` does not have an optional reduction parameter like :obj:`@parallel`. +@threadcall (Experimental) +-------------------------- +All I/O tasks, timers, REPL commands, etc are multiplexed onto a single OS thread via an event loop. +A patched version of libuv (http://docs.libuv.org/en/v1.x/) provides this functionality. Yield points provide +for co-operatively scheduling multiple tasks onto the same OS thread. I/O tasks and timers yield implicitly while +waiting for the event to occur. Calling `yield()` explicitly allows for other tasks to be scheduled. + +Thus, a task executing a ``ccall`` effectively prevents the Julia scheduler from executing any other +tasks till the call returns. This is true for all calls into external libraries. Exceptions are calls into +custom C code that call back into Julia (which may then yield) or C code that calls ``jl_yield()``(C equivalent of ``yield()``). + +Note that while Julia code runs on a single thread (by default), libraries used by Julia may launch their own internal +threads. For example, the BLAS library may start as many threads as there are cores on a machine. + +The ``@threadcall`` macro addresses scenarios where we do not want a ``ccall`` to block the main Julia event loop. +It schedules a C function for execution in a separate thread. A threadpool with a default size of 4 is used for this. +The size of the threadpool is controlled via environment variable UV_THREADPOOL_SIZE. While waiting for a free thread, +and during function execution once a thread is available, the requesting task (on the main Julia event loop) +yields to other tasks. Note that ``@threadcall`` does not return till the execution is complete. From a user point of +view, it is therefore a blocking call like other Julia API. + +It is very important that the called function does not call back into Julia. + +``@threadcall`` may be removed/changed in future versions of Julia. + .. rubric:: Footnotes .. [#mpi2rma] In this context, MPI refers to the MPI-1 standard. Beginning with MPI-2, the MPI standards committee introduced a new set of communication mechanisms, collectively referred to as Remote Memory Access (RMA). The motivation for adding RMA to the MPI standard was to facilitate one-sided communication patterns. For additional information on the latest MPI standard, see http://www.mpi-forum.org/docs. diff --git a/doc/stdlib/parallel.rst b/doc/stdlib/parallel.rst index 9aeaeb22d096f..8c37e3812cb46 100644 --- a/doc/stdlib/parallel.rst +++ b/doc/stdlib/parallel.rst @@ -11,7 +11,7 @@ Tasks .. Docstring generated from Julia source - Create a ``Task`` (i.e. thread, or coroutine) to execute the given function (which must be callable with no arguments). The task exits when this function returns. + Create a ``Task`` (i.e. coroutine) to execute the given function (which must be callable with no arguments). The task exits when this function returns. .. function:: yieldto(task, arg = nothing) @@ -814,6 +814,17 @@ will) change in the future. For further details, see LLVM's ``fence`` instruction. +ccall using a threadpool (Experimental) +--------------------------------------- + +.. function:: @threadcall((cfunc, clib), rettype, (argtypes...), argvals...) + + .. Docstring generated from Julia source + + The ``@threadcall`` macro is called in the same way as ``ccall`` but does the work in a different thread. This is useful when you want to call a blocking C function without causing the main ``julia`` thread to become blocked. Concurrency is limited by size of the libuv thread pool, which defaults to 4 threads but can be increased by setting the ``UV_THREADPOOL_SIZE`` environment variable and restarting the ``julia`` process. + + Note that the called function should never call back into Julia. + Synchronization Primitives -------------------------- From fe1c733f181a9b21d2431b3ef034350455649746 Mon Sep 17 00:00:00 2001 From: Simon Byrne Date: Tue, 9 Aug 2016 17:45:05 +0100 Subject: [PATCH 50/80] bump openlibm, fixes #17751 (#17912) (cherry picked from commit 008d8e752cad0cf2c5890221f738d6c9ebcd8087) --- .../md5 | 1 - .../sha512 | 1 - .../md5 | 1 + .../sha512 | 1 + deps/openlibm.version | 4 ++-- 5 files changed, 4 insertions(+), 4 deletions(-) delete mode 100644 deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/md5 delete mode 100644 deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/sha512 create mode 100644 deps/checksums/openlibm-71e79eb6f74d3f04ce724195b8ef5846a70d281e.tar.gz/md5 create mode 100644 deps/checksums/openlibm-71e79eb6f74d3f04ce724195b8ef5846a70d281e.tar.gz/sha512 diff --git a/deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/md5 b/deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/md5 deleted file mode 100644 index 8e7d4cc7c4029..0000000000000 --- a/deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -3b852052db9052a3668122449ce1f82c diff --git a/deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/sha512 b/deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/sha512 deleted file mode 100644 index a840095574b1a..0000000000000 --- a/deps/checksums/openlibm-0fa599cce845be67ed1729d9753f67e366c37b96.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -7125800186428a8aefea3030ff1d35f4169c29cb3ae30c2bd7965bd5e66f0958b8b33fcd2e5a30b012ecececd09ce3ce0195b5a8fc9066efd31e124cc32ec2c0 diff --git a/deps/checksums/openlibm-71e79eb6f74d3f04ce724195b8ef5846a70d281e.tar.gz/md5 b/deps/checksums/openlibm-71e79eb6f74d3f04ce724195b8ef5846a70d281e.tar.gz/md5 new file mode 100644 index 0000000000000..aa9ed22e8bb6d --- /dev/null +++ b/deps/checksums/openlibm-71e79eb6f74d3f04ce724195b8ef5846a70d281e.tar.gz/md5 @@ -0,0 +1 @@ +1bde34205734b9b30bd6a92768bd916b diff --git a/deps/checksums/openlibm-71e79eb6f74d3f04ce724195b8ef5846a70d281e.tar.gz/sha512 b/deps/checksums/openlibm-71e79eb6f74d3f04ce724195b8ef5846a70d281e.tar.gz/sha512 new file mode 100644 index 0000000000000..5af90ecd2b687 --- /dev/null +++ b/deps/checksums/openlibm-71e79eb6f74d3f04ce724195b8ef5846a70d281e.tar.gz/sha512 @@ -0,0 +1 @@ +9a324bf989fa513b6d513794114d08a24e77a4ff15da577c950c178322e34121f9eba6bf01e1520cd12a926723cda4687bb8efd6aa9de987cd9b3a1e6230d2ce diff --git a/deps/openlibm.version b/deps/openlibm.version index ceb185d9bc9de..af5b13cbea355 100644 --- a/deps/openlibm.version +++ b/deps/openlibm.version @@ -1,2 +1,2 @@ -OPENLIBM_BRANCH=v0.5.2 -OPENLIBM_SHA1=0fa599cce845be67ed1729d9753f67e366c37b96 +OPENLIBM_BRANCH=v0.5.3 +OPENLIBM_SHA1=71e79eb6f74d3f04ce724195b8ef5846a70d281e From 3f2a09dcf8e60a3dd07440d9f2c5a625741c3281 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Tue, 9 Aug 2016 17:54:51 -0700 Subject: [PATCH 51/80] Revert "Temporary ugly hack to download old versions of winrpm gcc dll's" (#17906) This reverts commit 97f18418b257bdcf625a5384687389b048e9c2ab. ref #15521 (cherry picked from commit 8a24871dcc2cd2b4175be198c159b7ec81b7a614) --- Makefile | 20 +++----------------- contrib/windows/msys_build.sh | 5 +++++ 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 7a5b562896b47..9d2aa47dd7527 100644 --- a/Makefile +++ b/Makefile @@ -468,7 +468,7 @@ endif ifeq ($(OS), WINNT) [ ! -d $(JULIAHOME)/dist-extras ] || ( cd $(JULIAHOME)/dist-extras && \ - cp 7z.exe 7z.dll busybox.exe libexpat-1.dll zlib1.dll libgfortran-3.dll libquadmath-0.dll libstdc++-6.dll libgcc_s_s*-1.dll libssp-0.dll $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin ) + cp 7z.exe 7z.dll busybox.exe libexpat-1.dll zlib1.dll $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin ) cd $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin && rm -f llvm* llc.exe lli.exe opt.exe LTO.dll bugpoint.exe macho-dump.exe # create file listing for uninstall. note: must have Windows path separators and line endings. @@ -593,14 +593,7 @@ ifneq (,$(filter $(ARCH), i386 i486 i586 i686)) $(JLDOWNLOAD) http://downloads.sourceforge.net/sevenzip/7z920.exe && \ 7z x -y 7z920.exe 7z.exe 7z.dll && \ ../contrib/windows/winrpm.sh http://download.opensuse.org/repositories/windows:/mingw:/win32/openSUSE_13.2 \ - "mingw32-libexpat1 mingw32-zlib1" && \ - $(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw32-libgfortran3-5.3.0-1.1.noarch.rpm && \ - $(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw32-libquadmath0-5.3.0-1.1.noarch.rpm && \ - $(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw32-libstdc%2B%2B6-5.3.0-1.1.noarch.rpm && \ - $(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw32-libgcc_s_sjlj1-5.3.0-1.1.noarch.rpm && \ - $(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw32-libssp0-5.3.0-1.1.noarch.rpm && \ - for i in *.rpm; do 7z x -y $$i; done && \ - for i in *.cpio; do 7z x -y $$i; done && \ + "mingw32-libgfortran3 mingw32-libquadmath0 mingw32-libstdc++6 mingw32-libgcc_s_sjlj1 mingw32-libssp0 mingw32-libexpat1 mingw32-zlib1" && \ cp usr/i686-w64-mingw32/sys-root/mingw/bin/*.dll . else ifeq ($(ARCH),x86_64) cd $(JULIAHOME)/dist-extras && \ @@ -609,14 +602,7 @@ else ifeq ($(ARCH),x86_64) mv _7z.dll 7z.dll && \ mv _7z.exe 7z.exe && \ ../contrib/windows/winrpm.sh http://download.opensuse.org/repositories/windows:/mingw:/win64/openSUSE_13.2 \ - "mingw64-libexpat1 mingw64-zlib1" && \ - $(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw64-libgfortran3-5.3.0-1.1.noarch.rpm && \ - $(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw64-libquadmath0-5.3.0-1.1.noarch.rpm && \ - $(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw64-libstdc%2B%2B6-5.3.0-1.1.noarch.rpm && \ - $(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw64-libgcc_s_seh1-5.3.0-1.1.noarch.rpm && \ - $(JLDOWNLOAD) https://juliacache.s3.amazonaws.com/mingw64-libssp0-5.3.0-1.1.noarch.rpm && \ - for i in *.rpm; do 7z x -y $$i; done && \ - for i in *.cpio; do 7z x -y $$i; done && \ + "mingw64-libgfortran3 mingw64-libquadmath0 mingw64-libstdc++6 mingw64-libgcc_s_seh1 mingw64-libssp0 mingw64-libexpat1 mingw64-zlib1" && \ cp usr/x86_64-w64-mingw32/sys-root/mingw/bin/*.dll . else $(error no win-extras target for ARCH=$(ARCH)) diff --git a/contrib/windows/msys_build.sh b/contrib/windows/msys_build.sh index 41258a21f414d..2d91f26a6af72 100755 --- a/contrib/windows/msys_build.sh +++ b/contrib/windows/msys_build.sh @@ -104,6 +104,11 @@ echo "override PCRE_INCL_PATH =" >> Make.user # Remove libjulia.dll if it was copied from downloaded binary rm -f usr/bin/libjulia.dll rm -f usr/bin/libjulia-debug.dll +rm -f usr/bin/libgcc_s_s*-1.dll +rm -f usr/bin/libgfortran-3.dll +rm -f usr/bin/libquadmath-0.dll +rm -f usr/bin/libssp-0.dll +rm -f usr/bin/libstdc++-6.dll if [ -z "$USEMSVC" ]; then if [ -z "`which ${CROSS_COMPILE}gcc 2>/dev/null`" -o -n "$APPVEYOR" ]; then From 9bc9a479de067ad94d8c014d70852a6b75413260 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Fri, 5 Aug 2016 14:37:25 -0400 Subject: [PATCH 52/80] use inferred type instead of `Union{}` for empty comprehensions fixes #17811 (cherry picked from commit 29e7ddfb3f6211347b34f6ba935e182bfb686d7e) ref #17847 --- base/array.jl | 15 ++++++++------- base/dict.jl | 11 +++++++++-- base/set.jl | 2 +- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/base/array.jl b/base/array.jl index 8c753b3e3924d..90b619ebd5f8f 100644 --- a/base/array.jl +++ b/base/array.jl @@ -283,14 +283,10 @@ function _collect(cont, itr, ::HasEltype, isz::SizeUnknown) end if isdefined(Core, :Inference) - function _default_eltype(itrt::ANY) - rt = Core.Inference.return_type(first, Tuple{itrt}) - return isleaftype(rt) ? rt : Union{} - end + _default_eltype(itrt::ANY) = Core.Inference.return_type(first, Tuple{itrt}) else - _default_eltype(itr::ANY) = Union{} + _default_eltype(itr::ANY) = Any end -_default_eltype{I,T}(::Type{Generator{I,Type{T}}}) = T _array_for(T, itr, ::HasLength) = Array{T,1}(Int(length(itr)::Integer)) _array_for(T, itr, ::HasShape) = similar(Array{T}, indices(itr)) @@ -354,7 +350,12 @@ function collect_to!{T}(dest::AbstractArray{T}, itr, offs, st) return dest end -function grow_to!(dest, itr, st = start(itr)) +function grow_to!(dest, itr) + out = grow_to!(similar(dest,Union{}), itr, start(itr)) + return isempty(out) ? dest : out +end + +function grow_to!(dest, itr, st) T = eltype(dest) while !done(itr, st) el, st = next(itr, st) diff --git a/base/dict.jl b/base/dict.jl index 8990bb198c1a2..2ea7e983b1c85 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -380,11 +380,18 @@ end dict_with_eltype{K,V}(kv, ::Type{Tuple{K,V}}) = Dict{K,V}(kv) dict_with_eltype{K,V}(kv, ::Type{Pair{K,V}}) = Dict{K,V}(kv) -dict_with_eltype(kv, t) = grow_to!(Dict{Union{},Union{}}(), kv) +dict_with_eltype{K,V}(::Type{Pair{K,V}}) = Dict{K,V}() +dict_with_eltype(::Type) = Dict() +dict_with_eltype(kv, t) = grow_to!(dict_with_eltype(_default_eltype(typeof(kv))), kv) # this is a special case due to (1) allowing both Pairs and Tuples as elements, # and (2) Pair being invariant. a bit annoying. -function grow_to!{K,V}(dest::Associative{K,V}, itr, st = start(itr)) +function grow_to!(dest::Associative, itr) + out = grow_to!(similar(dest, Pair{Union{},Union{}}), itr, start(itr)) + return isempty(out) ? dest : out +end + +function grow_to!{K,V}(dest::Associative{K,V}, itr, st) while !done(itr, st) (k,v), st = next(itr, st) if isa(k,K) && isa(v,V) diff --git a/base/set.jl b/base/set.jl index cafa9b66eeac1..804fffa8a4d3f 100644 --- a/base/set.jl +++ b/base/set.jl @@ -10,7 +10,7 @@ Set() = Set{Any}() Set(itr) = Set{eltype(itr)}(itr) function Set(g::Generator) T = _default_eltype(typeof(g)) - T === Union{} && return grow_to!(Set{T}(), g) + (isleaftype(T) || T === Union{}) || return grow_to!(Set{T}(), g) return Set{T}(g) end From 916d894f8cc535f9c9463633825368e8b3089779 Mon Sep 17 00:00:00 2001 From: Ranjan Anantharaman Date: Tue, 9 Aug 2016 20:27:00 +0530 Subject: [PATCH 53/80] test for #17811, type of e.g. `abs(Integer[])` (#17891) This tests that the result type of an empty comprehension is big enough to hold the possible results. (cherry picked from commit 93c60613d82c7d178b685d451b3aafc7cc30c624) --- test/inference.jl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/inference.jl b/test/inference.jl index dc8677df4d71e..2615e9cc4847a 100644 --- a/test/inference.jl +++ b/test/inference.jl @@ -274,3 +274,11 @@ end let f(x) = (x===nothing) ? 1 : 1.0 @test Base.return_types(f, (Void,)) == Any[Int] end + +# Issue #17811 +let I = Integer[] + I = abs(I) + @test typeof(I) == Array{Any,1} + push!(I, 1) + @test I == Any[1] +end From 45c90ef0a67917873c3cd96f630da2e479035aa4 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 5 Aug 2016 20:51:43 -0400 Subject: [PATCH 54/80] Refactor credentials yet again Now, we never prompt for credentials unless we consider the credentials invalid (we tried three times and gave up), (note exceptions are ssh keys which we know have to exist for ssh credentials to work). Also, if through interactive prompting credentials get changed, they get reset to make sure that we do the retries. This fixes the original issue of not prompting when a prompt would have been required as well as being nicer in the face of typos when entering credentials. Fixes certain clone operations over HTTPS. (cherry picked from commit dbbd20085f99a60566e9b994c045c8c44ea9724c) ref #17860 --- base/libgit2/callbacks.jl | 263 +++++++++++++++++++------------------- base/libgit2/libgit2.jl | 19 ++- base/libgit2/types.jl | 41 +++--- 3 files changed, 163 insertions(+), 160 deletions(-) diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index 0143125c3c2f7..a932923bbfe6c 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -48,90 +48,97 @@ function authenticate_ssh(creds::SSHCredentials, libgit2credptr::Ptr{Ptr{Void}}, err == 0 && return Cint(0) end - # if username is not provided, then prompt for it - username = if username_ptr == Cstring(C_NULL) - uname = creds.user # check if credentials were already used - uname !== nothing && !isusedcreds ? uname : prompt("Username for '$schema$host'") - else - unsafe_string(username_ptr) - end - creds.user = username # save credentials - isempty(username) && return Cint(Error.EAUTH) + if creds.prompt_if_incorrect + # if username is not provided, then prompt for it + username = if username_ptr == Cstring(C_NULL) + uname = creds.user # check if credentials were already used + !isusedcreds ? uname : prompt("Username for '$schema$host'", default=uname) + else + unsafe_string(username_ptr) + end + isempty(username) && return Cint(Error.EAUTH) - # For SSH we need a private key location - privatekey = if haskey(ENV,"SSH_KEY_PATH") - ENV["SSH_KEY_PATH"] - else - keydefpath = creds.prvkey # check if credentials were already used - keydefpath === nothing && (keydefpath = "") - if isempty(keydefpath) || isusedcreds - defaultkeydefpath = joinpath(homedir(),".ssh","id_rsa") - if isempty(keydefpath) && isfile(defaultkeydefpath) - keydefpath = defaultkeydefpath - else - keydefpath = - prompt("Private key location for '$schema$username@$host'", default=keydefpath) + # For SSH we need a private key location + privatekey = if haskey(ENV,"SSH_KEY_PATH") + ENV["SSH_KEY_PATH"] + else + keydefpath = creds.prvkey # check if credentials were already used + keydefpath === nothing && (keydefpath = "") + if isempty(keydefpath) || isusedcreds + defaultkeydefpath = joinpath(homedir(),".ssh","id_rsa") + if isempty(keydefpath) && isfile(defaultkeydefpath) + keydefpath = defaultkeydefpath + else + keydefpath = + prompt("Private key location for '$schema$username@$host'", default=keydefpath) + end end + keydefpath end - keydefpath - end - # If the private key changed, invalidate the cached public key - (privatekey != creds.prvkey) && - (creds.pubkey = "") - creds.prvkey = privatekey # save credentials + # If the private key changed, invalidate the cached public key + (privatekey != creds.prvkey) && + (creds.pubkey = "") - # For SSH we need a public key location, look for environment vars SSH_* as well - publickey = if haskey(ENV,"SSH_PUB_KEY_PATH") - ENV["SSH_PUB_KEY_PATH"] - else - keydefpath = creds.pubkey # check if credentials were already used - keydefpath === nothing && (keydefpath = "") - if isempty(keydefpath) || isusedcreds - if isempty(keydefpath) - keydefpath = privatekey*".pub" - end - if !isfile(keydefpath) - prompt("Public key location for '$schema$username@$host'", default=keydefpath) + # For SSH we need a public key location, look for environment vars SSH_* as well + publickey = if haskey(ENV,"SSH_PUB_KEY_PATH") + ENV["SSH_PUB_KEY_PATH"] + else + keydefpath = creds.pubkey # check if credentials were already used + keydefpath === nothing && (keydefpath = "") + if isempty(keydefpath) || isusedcreds + if isempty(keydefpath) + keydefpath = privatekey*".pub" + end + if !isfile(keydefpath) + prompt("Public key location for '$schema$username@$host'", default=keydefpath) + end end + keydefpath end - keydefpath - end - creds.pubkey = publickey # save credentials - passphrase_required = true - if !isfile(privatekey) - warn("Private key not found") - else - # In encrypted private keys, the second line is "Proc-Type: 4,ENCRYPTED" - open(privatekey) do f - passphrase_required = (readline(f); chomp(readline(f)) == "Proc-Type: 4,ENCRYPTED") + passphrase_required = true + if !isfile(privatekey) + warn("Private key not found") + else + # In encrypted private keys, the second line is "Proc-Type: 4,ENCRYPTED" + open(privatekey) do f + passphrase_required = (readline(f); chomp(readline(f)) == "Proc-Type: 4,ENCRYPTED") + end end - end - passphrase = if haskey(ENV,"SSH_KEY_PASS") - ENV["SSH_KEY_PASS"] - else - passdef = creds.pass # check if credentials were already used - passdef === nothing && (passdef = "") - if passphrase_required && (isempty(passdef) || isusedcreds) - if is_windows() - passdef = Base.winprompt( - "Your SSH Key requires a password, please enter it now:", - "Passphrase required", privatekey; prompt_username = false) - isnull(passdef) && return Cint(Error.EAUTH) - passdef = Base.get(passdef)[2] - else - passdef = prompt("Passphrase for $privatekey", password=true) + passphrase = if haskey(ENV,"SSH_KEY_PASS") + ENV["SSH_KEY_PASS"] + else + passdef = creds.pass # check if credentials were already used + passdef === nothing && (passdef = "") + if passphrase_required && (isempty(passdef) || isusedcreds) + if is_windows() + passdef = Base.winprompt( + "Your SSH Key requires a password, please enter it now:", + "Passphrase required", privatekey; prompt_username = false) + isnull(passdef) && return Cint(Error.EAUTH) + passdef = Base.get(passdef)[2] + else + passdef = prompt("Passphrase for $privatekey", password=true) + end end + passdef end - passdef + (creds.user != username) || (creds.pass != userpass) || + (creds.prvkey != privatekey) || (creds.pubkey != publickey) && reset!(creds) + + creds.user = username # save credentials + creds.prvkey = privatekey # save credentials + creds.pubkey = publickey # save credentials + creds.pass = passphrase + else + isusedcreds && return Cint(Error.EAUTH) end - creds.pass = passphrase err = ccall((:git_cred_ssh_key_new, :libgit2), Cint, (Ptr{Ptr{Void}}, Cstring, Cstring, Cstring, Cstring), - libgit2credptr, username, publickey, privatekey, passphrase) + libgit2credptr, creds.user, creds.pubkey, creds.prvkey, creds.pass) return err end @@ -139,33 +146,36 @@ function authenticate_userpass(creds::UserPasswordCredentials, libgit2credptr::P schema, host, urlusername) isusedcreds = checkused!(creds) - username = creds.user - userpass = creds.pass - if is_windows() - if username === nothing || userpass === nothing || isusedcreds - res = Base.winprompt("Please enter your credentials for '$schema$host'", "Credentials required", - username === nothing || isempty(username) ? - urlusername : username; prompt_username = true) - isnull(res) && return Cint(Error.EAUTH) - username, userpass = Base.get(res) - end - else - if username === nothing || isusedcreds - username = prompt("Username for '$schema$host'", default = urlusername) - end - - if userpass === nothing || isusedcreds + if creds.prompt_if_incorrect + username = creds.user + userpass = creds.pass + (username === nothing) && (username = "") + (userpass === nothing) && (userpass = "") + if is_windows() + if isempty(username) || isempty(userpass) || isusedcreds + res = Base.winprompt("Please enter your credentials for '$schema$host'", "Credentials required", + username === nothing || isempty(username) ? + urlusername : username; prompt_username = true) + isnull(res) && return Cint(Error.EAUTH) + username, userpass = Base.get(res) + end + elseif isusedcreds + username = prompt("Username for '$schema$host'", default = isempty(username) ? + urlusername : username) userpass = prompt("Password for '$schema$username@$host'", password=true) end - end - creds.user = username # save credentials - creds.pass = userpass # save credentials + (creds.user != username) || (creds.pass != userpass) && reset!(creds) + creds.user = username # save credentials + creds.pass = userpass # save credentials - isempty(username) && isempty(userpass) && return Cint(Error.EAUTH) + isempty(username) && isempty(userpass) && return Cint(Error.EAUTH) + else + isusedcreds && return Cint(Error.EAUTH) + end err = ccall((:git_cred_userpass_plaintext_new, :libgit2), Cint, (Ptr{Ptr{Void}}, Cstring, Cstring), - libgit2credptr, username, userpass) + libgit2credptr, creds.user, creds.pass) err == 0 && return Cint(0) end @@ -211,58 +221,41 @@ function credentials_callback(libgit2credptr::Ptr{Ptr{Void}}, url_ptr::Cstring, schema = schema === nothing ? "" : schema*"://" # get credentials object from payload pointer - creds = nothing - creds_are_temp = true - if payload_ptr != C_NULL - tmpobj = unsafe_pointer_to_objref(payload_ptr) - if isa(tmpobj, AbstractCredentials) - creds = tmpobj - creds_are_temp = false + @assert payload_ptr != C_NULL + creds = unsafe_pointer_to_objref(payload_ptr) + explicit = !isnull(creds[]) && !isa(Base.get(creds[]), CachedCredentials) + # use ssh key or ssh-agent + if isset(allowed_types, Cuint(Consts.CREDTYPE_SSH_KEY)) + sshcreds = get_creds!(creds, "ssh://$host", reset!(SSHCredentials(true), -1)) + if isa(sshcreds, SSHCredentials) + err = authenticate_ssh(sshcreds, libgit2credptr, username_ptr, schema, host) + err == 0 && return err end end - try - # use ssh key or ssh-agent - if isset(allowed_types, Cuint(Consts.CREDTYPE_SSH_KEY)) - sshcreds = get_creds!(creds, "ssh://$host", SSHCredentials()) - if isa(sshcreds, SSHCredentials) - creds = sshcreds # To make sure these get cleaned up below - err = authenticate_ssh(creds, libgit2credptr, username_ptr, schema, host) - err == 0 && return err - end - end - - if isset(allowed_types, Cuint(Consts.CREDTYPE_USERPASS_PLAINTEXT)) - defaultcreds = UserPasswordCredentials() - credid = "$schema$host" - upcreds = get_creds!(creds, credid, defaultcreds) - # If there were stored SSH credentials, but we ended up here that must - # mean that something went wrong. Replace the SSH credentials by user/pass - # credentials - if !isa(upcreds, UserPasswordCredentials) - upcreds = defaultcreds - isa(creds, CachedCredentials) && (creds.creds[credid] = upcreds) - end - creds = upcreds # To make sure these get cleaned up below - return authenticate_userpass(creds, libgit2credptr, schema, host, urlusername) + if isset(allowed_types, Cuint(Consts.CREDTYPE_USERPASS_PLAINTEXT)) + defaultcreds = reset!(UserPasswordCredentials(true), -1) + credid = "$schema$host" + upcreds = get_creds!(creds, credid, defaultcreds) + # If there were stored SSH credentials, but we ended up here that must + # mean that something went wrong. Replace the SSH credentials by user/pass + # credentials + if !isa(upcreds, UserPasswordCredentials) + upcreds = defaultcreds + isa(Base.get(creds[]), CachedCredentials) && (Base.get(creds[]).creds[credid] = upcreds) end + return authenticate_userpass(upcreds, libgit2credptr, schema, host, urlusername) + end - # No authentication method we support succeeded. The most likely cause is - # that explicit credentials were passed in, but said credentials are incompatible - # with the remote host. - if err == 0 - if (creds != nothing && !isa(creds, CachedCredentials)) - warn("The explicitly provided credentials were incompatible with " * - "the server's supported authentication methods") - end - err = Cint(Error.EAUTH) - end - finally - # if credentials are not passed back to caller via payload, - # then zero any passwords immediately. - if creds_are_temp && creds !== nothing - securezero!(creds) + # No authentication method we support succeeded. The most likely cause is + # that explicit credentials were passed in, but said credentials are incompatible + # with the remote host. + if err == 0 + if explicit + warn("The explicitly provided credentials were incompatible with " * + "the server's supported authentication methods") end + err = Cint(Error.EAUTH) end return Cint(err) end diff --git a/base/libgit2/libgit2.jl b/base/libgit2/libgit2.jl index fe9b08c87594c..321e974694966 100644 --- a/base/libgit2/libgit2.jl +++ b/base/libgit2/libgit2.jl @@ -143,18 +143,23 @@ function set_remote_url(path::AbstractString, url::AbstractString; remote::Abstr end end +function make_payload{P<:AbstractCredentials}(payload::Nullable{P}) + Ref{Nullable{AbstractCredentials}}(payload) +end + """ git fetch [|] []""" -function fetch{T<:AbstractString, P<:AbstractPayload}(repo::GitRepo; +function fetch{T<:AbstractString, P<:AbstractCredentials}(repo::GitRepo; remote::AbstractString="origin", remoteurl::AbstractString="", refspecs::Vector{T}=AbstractString[], - payload::Nullable{P}=Nullable{AbstractPayload}(SSHCredentials())) + payload::Nullable{P}=Nullable{AbstractCredentials}()) rmt = if isempty(remoteurl) get(GitRemote, repo, remote) else GitRemoteAnon(repo, remoteurl) end try + payload = make_payload(payload) fo = FetchOptions(callbacks=RemoteCallbacks(credentials_cb(), payload)) fetch(rmt, refspecs, msg="from $(url(rmt))", options = fo) finally @@ -163,18 +168,19 @@ function fetch{T<:AbstractString, P<:AbstractPayload}(repo::GitRepo; end """ git push [|] []""" -function push{T<:AbstractString, P<:AbstractPayload}(repo::GitRepo; +function push{T<:AbstractString, P<:AbstractCredentials}(repo::GitRepo; remote::AbstractString="origin", remoteurl::AbstractString="", refspecs::Vector{T}=AbstractString[], force::Bool=false, - payload::Nullable{P}=Nullable{AbstractPayload}(SSHCredentials())) + payload::Nullable{P}=Nullable{AbstractCredentials}()) rmt = if isempty(remoteurl) get(GitRemote, repo, remote) else GitRemoteAnon(repo, remoteurl) end try + payload = make_payload(payload) push_opts=PushOptions(callbacks=RemoteCallbacks(credentials_cb(), payload)) push(rmt, refspecs, force=force, options=push_opts) finally @@ -303,13 +309,14 @@ function checkout!(repo::GitRepo, commit::AbstractString = ""; end """ git clone [-b ] [--bare] """ -function clone{P<:AbstractPayload}(repo_url::AbstractString, repo_path::AbstractString; +function clone{P<:AbstractCredentials}(repo_url::AbstractString, repo_path::AbstractString; branch::AbstractString="", isbare::Bool = false, remote_cb::Ptr{Void} = C_NULL, - payload::Nullable{P}=Nullable{AbstractPayload}(SSHCredentials())) + payload::Nullable{P}=Nullable{AbstractCredentials}()) # setup clone options lbranch = Base.cconvert(Cstring, branch) + payload = make_payload(payload) fetch_opts=FetchOptions(callbacks = RemoteCallbacks(credentials_cb(), payload)) clone_opts = CloneOptions( bare = Cint(isbare), diff --git a/base/libgit2/types.jl b/base/libgit2/types.jl index c5ce76c79fb8c..36b01fa0f46ac 100644 --- a/base/libgit2/types.jl +++ b/base/libgit2/types.jl @@ -50,11 +50,9 @@ function Base.finalize(buf::Buffer) return buf_ptr[] end -"Abstract payload type for callback functions" -abstract AbstractPayload - "Abstract credentials payload" -abstract AbstractCredentials <: AbstractPayload +abstract AbstractCredentials + "Checks if credentials were used" checkused!(p::AbstractCredentials) = true checkused!(p::Void) = false @@ -172,13 +170,8 @@ RemoteCallbacks(; sideband_progress::Ptr{Void} = C_NULL, transport, payload) -function RemoteCallbacks{P<:AbstractPayload}(credentials::Ptr{Void}, payload::Nullable{P}) - if isnull(payload) - RemoteCallbacks(credentials=credentials_cb()) - else - payload_ptr = pointer_from_objref(Base.get(payload)) - RemoteCallbacks(credentials=credentials_cb(), payload=payload_ptr) - end +function RemoteCallbacks(credentials::Ptr{Void}, payload::Ref{Nullable{AbstractCredentials}}) + RemoteCallbacks(credentials=credentials_cb(), payload=pointer_from_objref(payload)) end if LibGit2.version() >= v"0.24.0" @@ -698,13 +691,14 @@ import Base.securezero! type UserPasswordCredentials <: AbstractCredentials user::String pass::String + prompt_if_incorrect::Bool # Whether to allow interactive prompting if the credentials are incorrect count::Int # authentication failure protection count - function UserPasswordCredentials(u::AbstractString,p::AbstractString) - c = new(u,p,3) + function UserPasswordCredentials(u::AbstractString,p::AbstractString,prompt_if_incorrect::Bool=false) + c = new(u,p,prompt_if_incorrect,3) finalizer(c, securezero!) return c end - UserPasswordCredentials() = UserPasswordCredentials("","") + UserPasswordCredentials(prompt_if_incorrect::Bool=false) = UserPasswordCredentials("","",prompt_if_incorrect) end function securezero!(cred::UserPasswordCredentials) @@ -721,14 +715,15 @@ type SSHCredentials <: AbstractCredentials pubkey::String prvkey::String usesshagent::String # used for ssh-agent authentication + prompt_if_incorrect::Bool # Whether to allow interactive prompting if the credentials are incorrect count::Int - function SSHCredentials(u::AbstractString,p::AbstractString) - c = new(u,p,"","","Y",3) + function SSHCredentials(u::AbstractString,p::AbstractString,prompt_if_incorrect::Bool=false) + c = new(u,p,"","","Y",prompt_if_incorrect,3) finalizer(c, securezero!) return c end - SSHCredentials() = SSHCredentials("","") + SSHCredentials(prompt_if_incorrect::Bool=false) = SSHCredentials("","",prompt_if_incorrect) end function securezero!(cred::SSHCredentials) securezero!(cred.user) @@ -752,13 +747,21 @@ function checkused!(p::Union{UserPasswordCredentials, SSHCredentials}) p.count -= 1 return false end -reset!(p::Union{UserPasswordCredentials, SSHCredentials}, cnt::Int=3) = (p.count = cnt) -reset!(p::CachedCredentials) = foreach(reset!, values(p.cred)) +reset!(p::Union{UserPasswordCredentials, SSHCredentials}, cnt::Int=3) = (p.count = cnt; p) +reset!(p::CachedCredentials) = (foreach(reset!, values(p.cred)); p) "Obtain the cached credentials for the given host+protocol (credid), or return and store the default if not found" get_creds!(collection::CachedCredentials, credid, default) = get!(collection.cred, credid, default) get_creds!(creds::AbstractCredentials, credid, default) = creds get_creds!(creds::Void, credid, default) = default +function get_creds!(creds::Ref{Nullable{AbstractCredentials}}, credid, default) + if isnull(creds[]) + creds[] = Nullable{AbstractCredentials}(default) + return default + else + get_creds!(Base.get(creds[]), credid, default) + end +end function securezero!(p::CachedCredentials) foreach(securezero!, values(p.cred)) From c0bf5731c2e173bcfdf5a351321d3125a836968f Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Tue, 9 Aug 2016 10:05:51 -0400 Subject: [PATCH 55/80] Don't print an error message if ssh-agent fails Since we'll just fall back to the keys anyway. (cherry picked from commit a1cdc5409dfca99b9380e549793adf59c6330fb5) ref #17860 --- base/libgit2/callbacks.jl | 4 ---- 1 file changed, 4 deletions(-) diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index a932923bbfe6c..91a0b8d4ca42b 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -32,12 +32,8 @@ function authenticate_ssh(creds::SSHCredentials, libgit2credptr::Ptr{Ptr{Void}}, if errcls != Error.None # Check if we used ssh-agent if creds.usesshagent == "U" - println("ERROR: $errmsg ssh-agent") creds.usesshagent = "E" # reported ssh-agent error, disables ssh agent use for the future - else - println("ERROR: $errmsg") end - flush(STDOUT) end # first try ssh-agent if credentials support its usage From b6372abb2cf08faac4259a60cb0fc200672bc4a9 Mon Sep 17 00:00:00 2001 From: kshyatt Date: Fri, 5 Aug 2016 16:59:08 -0700 Subject: [PATCH 56/80] More examples, cleanup of array and eigen method docs (cherry picked from commit 24cdc2675623ea85e1260adaed365ba1e095c081) ref #17882 --- base/abstractarray.jl | 77 +++++++- base/array.jl | 388 ++++++++++++++++++++++++++++++++++++- base/collections.jl | 78 +++++++- base/datafmt.jl | 10 +- base/docs/helpdb/Base.jl | 322 ------------------------------ base/linalg/eigen.jl | 205 +++++++++++++++++++- base/reducedim.jl | 10 + doc/stdlib/arrays.rst | 308 ++++++++++++++++++++++++++--- doc/stdlib/collections.rst | 122 ++++++++++-- doc/stdlib/io-network.rst | 6 +- doc/stdlib/linalg.rst | 74 ++++++- 11 files changed, 1197 insertions(+), 403 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index dd58280e1de5f..74dc06ccccf68 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -26,6 +26,23 @@ function vect(X...) copy!(Array{T,1}(length(X)), X) end +""" + size(A::AbstractArray, [dim...]) + +Returns a tuple containing the dimensions of `A`. Optionally you can specify the +dimension(s) you want the length of, and get the length of that dimension, or a tuple of the +lengths of dimensions you asked for. + +```jldoctest +julia> A = ones(2,3,4); + +julia> size(A, 2) +3 + +julia> size(A,3,2) +(4,3) +``` +""" size{T,N}(t::AbstractArray{T,N}, d) = d <= N ? size(t)[d] : 1 size{N}(x, d1::Integer, d2::Integer, dx::Vararg{Integer, N}) = (size(x, d1), size(x, d2), ntuple(k->size(x, dx[k]), Val{N})...) @@ -72,9 +89,34 @@ linearindices(A::AbstractVector) = (@_inline_meta; indices1(A)) eltype{T}(::Type{AbstractArray{T}}) = T eltype{T,N}(::Type{AbstractArray{T,N}}) = T elsize{T}(::AbstractArray{T}) = sizeof(T) +""" + ndims(A::AbstractArray) -> Integer + +Returns the number of dimensions of `A`. + +```jldoctest +julia> A = ones(3,4,5); + +julia> ndims(A) +3 +``` +""" ndims{T,N}(::AbstractArray{T,N}) = N ndims{T,N}(::Type{AbstractArray{T,N}}) = N ndims{T<:AbstractArray}(::Type{T}) = ndims(supertype(T)) + +""" + length(A::AbstractArray) -> Integer + +Returns the number of elements in `A`. + +```jldoctest +julia> A = ones(3,4,5); + +julia> length(A) +60 +``` +""" length(t::AbstractArray) = prod(size(t)) _length(A::AbstractArray) = prod(map(unsafe_length, indices(A))) # circumvent missing size _length(A) = length(A) @@ -89,9 +131,19 @@ end last(a) = a[end] """ - stride(A, k) + stride(A, k::Integer) Returns the distance in memory (in number of elements) between adjacent elements in dimension `k`. + +```jldoctest +julia> A = ones(3,4,5); + +julia> stride(A,2) +3 + +julia> stride(A,3) +12 +``` """ function stride(a::AbstractArray, i::Integer) if i > ndims(a) @@ -109,6 +161,13 @@ strides{T}(A::AbstractArray{T,0}) = () strides(A) Returns a tuple of the memory strides in each dimension. + +```jldoctest +julia> A = ones(3,4,5); + +julia> strides(A) +(1,3,12) +``` """ strides(A::AbstractArray) = _strides((1,), A) _strides{T,N}(out::NTuple{N}, A::AbstractArray{T,N}) = out @@ -146,6 +205,22 @@ abstract LinearIndexing immutable LinearFast <: LinearIndexing end immutable LinearSlow <: LinearIndexing end +""" + Base.linearindexing(A) + +`linearindexing` defines how an AbstractArray most efficiently accesses its elements. If +`Base.linearindexing(A)` returns `Base.LinearFast()`, this means that linear indexing with +only one index is an efficient operation. If it instead returns `Base.LinearSlow()` (by +default), this means that the array intrinsically accesses its elements with indices +specified for every dimension. Since converting a linear index to multiple indexing +subscripts is typically very expensive, this provides a traits-based mechanism to enable +efficient generic code for all array types. + +An abstract array subtype `MyArray` that wishes to opt into fast linear indexing behaviors +should define `linearindexing` in the type-domain: + + Base.linearindexing{T<:MyArray}(::Type{T}) = Base.LinearFast() +""" linearindexing(A::AbstractArray) = linearindexing(typeof(A)) linearindexing{T<:AbstractArray}(::Type{T}) = LinearSlow() linearindexing{T<:Array}(::Type{T}) = LinearFast() diff --git a/base/array.jl b/base/array.jl index 90b619ebd5f8f..a24a7773477a7 100644 --- a/base/array.jl +++ b/base/array.jl @@ -555,6 +555,22 @@ function insert!{T}(a::Array{T,1}, i::Integer, item) return a end +""" + deleteat!(a::Vector, i::Integer) + +Remove the item at the given `i` and return the modified `a`. Subsequent items +are shifted to fill the resulting gap. + +```jldoctest +julia> deleteat!([6, 5, 4, 3, 2, 1], 2) +5-element Array{Int64,1}: + 6 + 4 + 3 + 2 + 1 +``` +""" deleteat!(a::Vector, i::Integer) = (_deleteat!(a, i, 1); a) function deleteat!{T<:Integer}(a::Vector, r::UnitRange{T}) @@ -563,6 +579,25 @@ function deleteat!{T<:Integer}(a::Vector, r::UnitRange{T}) return a end +""" + deleteat!(a::Vector, inds) + +Remove the items at the indices given by `inds`, and return the modified `a`. +Subsequent items are shifted to fill the resulting gap. `inds` must be sorted and unique. + +```jldoctest +julia> deleteat!([6, 5, 4, 3, 2, 1], 1:2:5) +3-element Array{Int64,1}: + 5 + 3 + 1 + +julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) +ERROR: ArgumentError: indices must be unique and sorted + in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:576 + ... +``` +""" function deleteat!(a::Vector, inds) n = length(a) s = start(inds) @@ -733,7 +768,24 @@ end ## find ## -# returns the index of the next non-zero element, or 0 if all zeros +""" + findnext(A, i::Integer) + +Find the next linear index >= `i` of a non-zero element of `A`, or `0` if not found. + +```jldoctest +julia> A = [0 0; 1 0] +2×2 Array{Int64,2}: + 0 0 + 1 0 + +julia> findnext(A,1) +2 + +julia> findnext(A,3) +0 +``` +""" function findnext(A, start::Integer) for i = start:length(A) if A[i] != 0 @@ -742,9 +794,43 @@ function findnext(A, start::Integer) end return 0 end + +""" + findfirst(A) + +Return the linear index of the first non-zero value in `A` (determined by `A[i]!=0`). +Returns `0` if no such value is found. + +```jldoctest +julia> A = [0 0; 1 0] +2×2 Array{Int64,2}: + 0 0 + 1 0 + +julia> findfirst(A) +2 +``` +""" findfirst(A) = findnext(A, 1) -# returns the index of the next matching element +""" + findnext(A, v, i::Integer) + +Find the next linear index >= `i` of an element of `A` equal to `v` (using `==`), or `0` if not found. + +```jldoctest +julia> A = [1 4; 2 2] +2×2 Array{Int64,2}: + 1 4 + 2 2 + +julia> findnext(A,4,4) +0 + +julia> findnext(A,4,3) +3 +``` +""" function findnext(A, v, start::Integer) for i = start:length(A) if A[i] == v @@ -753,9 +839,45 @@ function findnext(A, v, start::Integer) end return 0 end +""" + findfirst(A, v) + +Return the linear index of the first element equal to `v` in `A`. +Returns `0` if `v` is not found. + +```jldoctest +julia> A = [4 6; 2 2] +2×2 Array{Int64,2}: + 4 6 + 2 2 + +julia> findfirst(A,2) +2 + +julia> findfirst(A,3) +0 +``` +""" findfirst(A, v) = findnext(A, v, 1) -# returns the index of the next element for which the function returns true +""" + findnext(predicate::Function, A, i::Integer) + +Find the next linear index >= `i` of an element of `A` for which `predicate` returns `true`, or `0` if not found. + +```jldoctest +julia> A = [1 4; 2 2] +2×2 Array{Int64,2}: + 1 4 + 2 2 + +julia> findnext(isodd, A, 1) +1 + +julia> findnext(isodd, A, 2) +0 +``` +""" function findnext(testf::Function, A, start::Integer) for i = start:length(A) if testf(A[i]) @@ -764,35 +886,193 @@ function findnext(testf::Function, A, start::Integer) end return 0 end + +""" + findfirst(predicate::Function, A) + +Return the linear index of the first element of `A` for which `predicate` returns `true`. +Returns `0` if there is no such element. + +```jldoctest +julia> A = [1 4; 2 2] +2×2 Array{Int64,2}: + 1 4 + 2 2 + +julia> findfirst(iseven, A) +2 + +julia> findfirst(x -> x>10, A) +0 +``` +""" findfirst(testf::Function, A) = findnext(testf, A, 1) -# returns the index of the previous non-zero element, or 0 if all zeros +""" + findprev(A, i::Integer) + +Find the previous linear index <= `i` of a non-zero element of `A`, or `0` if not found. + +```jldoctest +julia> A = [0 0; 1 2] +2×2 Array{Int64,2}: + 0 0 + 1 2 + +julia> findprev(A,2) +2 + +julia> findprev(A,1) +0 +``` +""" function findprev(A, start::Integer) for i = start:-1:1 A[i] != 0 && return i end return 0 end + +""" + findlast(A) + +Return the linear index of the last non-zero value in `A` (determined by `A[i]!=0`). +Returns `0` if there is no non-zero value in `A`. + +```jldoctest +julia> A = [1 0; 1 0] +2×2 Array{Int64,2}: + 1 0 + 1 0 + +julia> findlast(A) +2 + +julia> A = zeros(2,2) +2×2 Array{Float64,2}: + 0.0 0.0 + 0.0 0.0 + +julia> findlast(A) +0 +``` +""" findlast(A) = findprev(A, length(A)) -# returns the index of the matching element, or 0 if no matching +""" + findprev(A, v, i::Integer) + +Find the previous linear index <= `i` of an element of `A` equal to `v` (using `==`), or `0` if not found. + +```jldoctest +julia> A = [0 0; 1 2] +2×2 Array{Int64,2}: + 0 0 + 1 2 + +julia> findprev(A, 1, 4) +2 + +julia> findprev(A, 1, 1) +0 +``` +""" function findprev(A, v, start::Integer) for i = start:-1:1 A[i] == v && return i end return 0 end + +""" + findlast(A, v) + +Return the linear index of the last element equal to `v` in `A`. +Returns `0` if there is no element of `A` equal to `v`. + +```jldoctest +julia> A = [1 2; 2 1] +2×2 Array{Int64,2}: + 1 2 + 2 1 + +julia> findlast(A,1) +4 + +julia> findlast(A,2) +3 + +julia> findlast(A,3) +0 +``` +""" findlast(A, v) = findprev(A, v, length(A)) -# returns the index of the previous element for which the function returns true, or zero if it never does +""" + findprev(predicate::Function, A, i::Integer) + +Find the previous linear index <= `i` of an element of `A` for which `predicate` returns `true`, or +`0` if not found. + +```jldoctest +julia> A = [4 6; 1 2] +2×2 Array{Int64,2}: + 4 6 + 1 2 + +julia> findprev(isodd, A, 1) +0 + +julia> findprev(isodd, A, 3) +2 +``` +""" function findprev(testf::Function, A, start::Integer) for i = start:-1:1 testf(A[i]) && return i end return 0 end + +""" + findlast(predicate::Function, A) + +Return the linear index of the last element of `A` for which `predicate` returns `true`. +Returns `0` if there is no such element. + +```jldoctest +julia> A = [1 2; 3 4] +2×2 Array{Int64,2}: + 1 2 + 3 4 + +julia> findlast(isodd, A) +2 + +julia> findlast(x -> x > 5, A) +0 +``` +""" findlast(testf::Function, A) = findprev(testf, A, length(A)) +""" + find(f::Function, A) + +Return a vector `I` of the linear indexes of `A` where `f(A[I])` returns `true`. +If there are no such elements of `A`, find returns an empty array. + +```jldoctest +julia> A = [1 2; 3 4] +2×2 Array{Int64,2}: + 1 2 + 3 4 + +julia> find(isodd,A) +2-element Array{Int64,1}: + 1 + 2 +``` +""" function find(testf::Function, A) # use a dynamic-length array to store the indexes, then copy to a non-padded # array for the return @@ -807,6 +1087,25 @@ function find(testf::Function, A) return I end +""" + find(A) + +Return a vector of the linear indexes of the non-zeros in `A` (determined by `A[i]!=0`). A +common use of this is to convert a boolean array to an array of indexes of the `true` +elements. If there are no non-zero elements of `A`, `find` returns an empty array. + +```jldoctest +julia> A = [true false; false true] +2×2 Array{Bool,2}: + true false + false true + +julia> find(A) +2-element Array{Int64,1}: + 1 + 4 +``` +""" function find(A) nnzA = countnz(A) I = Vector{Int}(nnzA) @@ -825,6 +1124,32 @@ find(testf::Function, x::Number) = !testf(x) ? Array{Int,1}(0) : [1] findn(A::AbstractVector) = find(A) +""" + findn(A) + +Return a vector of indexes for each dimension giving the locations of the non-zeros in `A` +(determined by `A[i]!=0`). +If there are no non-zero elements of `A`, `findn` returns a 2-tuple of empty arrays. + +```jldoctest +julia> A = [1 2 0; 0 0 3; 0 4 0] +3×3 Array{Int64,2}: + 1 2 0 + 0 0 3 + 0 4 0 + +julia> findn(A) +([1,1,3,2],[1,2,2,3]) + +julia> A = zeros(2,2) +2×2 Array{Float64,2}: + 0.0 0.0 + 0.0 0.0 + +julia> findn(A) +(Int64[],Int64[]) +``` +""" function findn(A::AbstractMatrix) nnzA = countnz(A) I = similar(A, Int, nnzA) @@ -840,6 +1165,23 @@ function findn(A::AbstractMatrix) return (I, J) end +""" + findnz(A) + +Return a tuple `(I, J, V)` where `I` and `J` are the row and column indexes of the non-zero +values in matrix `A`, and `V` is a vector of the non-zero values. + +```jldoctest +julia> A = [1 2 0; 0 0 3; 0 4 0] +3×3 Array{Int64,2}: + 1 2 0 + 0 0 3 + 0 4 0 + +julia> findnz(A) +([1,1,3,2],[1,2,2,3],[1,2,4,3]) +``` +""" function findnz{T}(A::AbstractMatrix{T}) nnzA = countnz(A) I = zeros(Int, nnzA) @@ -922,6 +1264,8 @@ end indmax(itr) -> Integer Returns the index of the maximum element in a collection. +The collection must not be empty. + ```jldoctest julia> indmax([8,0.1,-9,pi]) 1 @@ -933,6 +1277,8 @@ indmax(a) = findmax(a)[2] indmin(itr) -> Integer Returns the index of the minimum element in a collection. +The collection must not be empty. + ```jldoctest julia> indmin([8,0.1,-9,pi]) 3 @@ -1092,6 +1438,22 @@ function union(vs...) ret end # setdiff only accepts two args + +""" + setdiff(a, b) + +Construct the set of elements in `a` but not `b`. Maintains order with arrays. Note that +both arguments must be collections, and both will be iterated over. In particular, +`setdiff(set,element)` where `element` is a potential member of `set`, will not work in +general. + +```jldoctest +julia> setdiff([1,2,3],[3,4,5]) +2-element Array{Int64,1}: + 1 + 2 +``` +""" function setdiff(a, b) args_type = promote_type(eltype(a), eltype(b)) bset = Set(b) @@ -1112,4 +1474,18 @@ end # store counts with a Dict. symdiff(a) = a symdiff(a, b) = union(setdiff(a,b), setdiff(b,a)) +""" + symdiff(a, b, rest...) + +Construct the symmetric difference of elements in the passed in sets or arrays. +Maintains order with arrays. + +```jldoctest +julia> symdiff([1,2,3],[3,4,5],[4,5,6]) +3-element Array{Int64,1}: + 1 + 2 + 6 +``` +""" symdiff(a, b, rest...) = symdiff(a, symdiff(b, rest...)) diff --git a/base/collections.jl b/base/collections.jl index 545112e25ba8b..caeebf673fa86 100644 --- a/base/collections.jl +++ b/base/collections.jl @@ -109,7 +109,7 @@ end # Turn an arbitrary array into a binary min-heap in linear time. """ - heapify!(v, [ord]) + heapify!(v, ord::Ordering=Forward) In-place [`heapify`](:func:`heapify`). """ @@ -121,16 +121,49 @@ function heapify!(xs::AbstractArray, o::Ordering=Forward) end """ - heapify(v, [ord]) + heapify(v, ord::Ordering=Forward) Returns a new vector in binary heap order, optionally using the given ordering. +```jldoctest +julia> a = [1,3,4,5,2]; + +julia> Base.Collections.heapify(a) +5-element Array{Int64,1}: + 1 + 2 + 4 + 5 + 3 + +julia> Base.Collections.heapify(a, Base.Order.Reverse) +5-element Array{Int64,1}: + 5 + 3 + 4 + 1 + 2 +``` """ heapify(xs::AbstractArray, o::Ordering=Forward) = heapify!(copymutable(xs), o) """ - isheap(v, [ord]) + isheap(v, ord::Ordering=Forward) Return `true` if an array is heap-ordered according to the given order. + +```jldoctest +julia> a = [1,2,3] +3-element Array{Int64,1}: + 1 + 2 + 3 + +julia> Base.Collections.isheap(a,Base.Order.Forward) +true + +julia> Base.Collections.isheap(a,Base.Order.Reverse) +false +``` """ function isheap(xs::AbstractArray, o::Ordering=Forward) for i in 1:div(length(xs), 2) @@ -157,6 +190,14 @@ the default comparison for `V`. A `PriorityQueue` acts like a `Dict`, mapping values to their priorities, with the addition of a `dequeue!` function to remove the lowest priority element. + +```jldoctest +julia> a = Base.Collections.PriorityQueue(["a","b","c"],[2,3,1],Base.Order.Forward) +Base.Collections.PriorityQueue{String,Int64,Base.Order.ForwardOrdering} with 3 entries: + "c" => 1 + "b" => 3 + "a" => 2 +``` """ type PriorityQueue{K,V,O<:Ordering} <: Associative{K,V} # Binary heap of (element, priority) pairs. @@ -304,6 +345,21 @@ end enqueue!(pq, k, v) Insert the a key `k` into a priority queue `pq` with priority `v`. + +```jldoctest +julia> a = Base.Collections.PriorityQueue(["a","b","c"],[2,3,1],Base.Order.Forward) +Base.Collections.PriorityQueue{String,Int64,Base.Order.ForwardOrdering} with 3 entries: + "c" => 1 + "b" => 3 + "a" => 2 + +julia> Base.Collections.enqueue!(a, "d", 4) +Base.Collections.PriorityQueue{String,Int64,Base.Order.ForwardOrdering} with 4 entries: + "c" => 1 + "b" => 3 + "a" => 2 + "d" => 4 +``` """ function enqueue!{K,V}(pq::PriorityQueue{K,V}, key, value) if haskey(pq, key) @@ -319,6 +375,22 @@ end dequeue!(pq) Remove and return the lowest priority key from a priority queue. + +```jldoctest +julia> a = Base.Collections.PriorityQueue(["a","b","c"],[2,3,1],Base.Order.Forward) +Base.Collections.PriorityQueue{String,Int64,Base.Order.ForwardOrdering} with 3 entries: + "c" => 1 + "b" => 3 + "a" => 2 + +julia> Base.Collections.dequeue!(a) +"c" + +julia> a +Base.Collections.PriorityQueue{String,Int64,Base.Order.ForwardOrdering} with 2 entries: + "b" => 3 + "a" => 2 +``` """ function dequeue!(pq::PriorityQueue) x = pq.xs[1] diff --git a/base/datafmt.jl b/base/datafmt.jl index 7016d09a3d6d3..13efa2c337277 100644 --- a/base/datafmt.jl +++ b/base/datafmt.jl @@ -690,11 +690,11 @@ function writedlm(fname::AbstractString, a, dlm; opts...) end """ - writedlm(f, A, dl='\\t'; opts) + writedlm(f, A, delim='\\t'; opts) -Write `A` (a vector, matrix or an iterable collection of iterable rows) as text to `f` -(either a filename string or an `IO` stream) using the given delimiter `delim` (which -defaults to tab, but can be any printable Julia object, typically a `Char` or +Write `A` (a vector, matrix, or an iterable collection of iterable rows) as text to `f` +(either a filename string or an [`IO`](:class:`IO`) stream) using the given delimiter +`delim` (which defaults to tab, but can be any printable Julia object, typically a `Char` or `AbstractString`). For example, two vectors `x` and `y` of the same length can be written as two columns of @@ -705,7 +705,7 @@ writedlm(io, a; opts...) = writedlm(io, a, '\t'; opts...) """ writecsv(filename, A; opts) -Equivalent to `writedlm` with `delim` set to comma. +Equivalent to [`writedlm`](:func:`writedlm`) with `delim` set to comma. """ writecsv(io, a; opts...) = writedlm(io, a, ','; opts...) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 2175bd10b0a16..3a669992c71f3 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -263,14 +263,6 @@ The text is assumed to be encoded in UTF-8. """ readlines -""" - findnz(A) - -Return a tuple `(I, J, V)` where `I` and `J` are the row and column indexes of the non-zero -values in matrix `A`, and `V` is a vector of the non-zero values. -""" -findnz - """ foldl(op, v0, itr) @@ -1639,13 +1631,6 @@ Bitwise and. """ & -""" - eigmax(A) - -Returns the largest eigenvalue of `A`. -""" -eigmax - """ PipeBuffer() @@ -2113,21 +2098,6 @@ themselves in another collection. The result is of the preceding example is equi """ append! -""" - find(A) - -Return a vector of the linear indexes of the non-zeros in `A` (determined by `A[i]!=0`). A -common use of this is to convert a boolean array to an array of indexes of the `true` -elements. -""" -find(A) - -""" - find(f,A) - -Return a vector of the linear indexes of `A` where `f` returns `true`. -""" -find(f, A) """ ctranspose(A) @@ -2309,24 +2279,6 @@ Multiply elements of an array over the given dimensions. """ prod(A, dims) -""" - Base.linearindexing(A) - -`linearindexing` defines how an AbstractArray most efficiently accesses its elements. If -`Base.linearindexing(A)` returns `Base.LinearFast()`, this means that linear indexing with -only one index is an efficient operation. If it instead returns `Base.LinearSlow()` (by -default), this means that the array intrinsically accesses its elements with indices -specified for every dimension. Since converting a linear index to multiple indexing -subscripts is typically very expensive, this provides a traits-based mechanism to enable -efficient generic code for all array types. - -An abstract array subtype `MyArray` that wishes to opt into fast linear indexing behaviors -should define `linearindexing` in the type-domain: - - Base.linearindexing{T<:MyArray}(::Type{T}) = Base.LinearFast() -""" -Base.linearindexing - """ isqrt(n) @@ -3215,14 +3167,6 @@ results `A[ks...]`, where `ks` goes over the positions in the broadcast. """ broadcast_getindex -""" - findn(A) - -Return a vector of indexes for each dimension giving the locations of the non-zeros in `A` -(determined by `A[i]!=0`). -""" -findn - """ invoke(f, (types...), args...) @@ -3380,13 +3324,6 @@ Display an informational message. Argument `msg` is a string describing the info """ info -""" - eigmin(A) - -Returns the smallest eigenvalue of `A`. -""" -eigmin - """ ltoh(x) @@ -4188,13 +4125,6 @@ last argument optionally specifies a size beyond which the buffer may not be gro """ IOBuffer(data=?) -""" - findmax(A, dims) -> (maxval, index) - -For an array input, returns the value and index of the maximum over the given dimensions. -""" -findmax(A,dims) - """ tempname() @@ -4274,45 +4204,6 @@ Compute the inverse hyperbolic tangent of `x`. """ atanh -""" - deleteat!(collection, index) - -Remove the item at the given `index` and return the modified `collection`. Subsequent items -are shifted to fill the resulting gap. - -```jldoctest -julia> deleteat!([6, 5, 4, 3, 2, 1], 2) -5-element Array{Int64,1}: - 6 - 4 - 3 - 2 - 1 -``` -""" -deleteat!(collection, index::Integer) - -""" - deleteat!(collection, itr) - -Remove the items at the indices given by `itr`, and return the modified `collection`. -Subsequent items are shifted to fill the resulting gap. `itr` must be sorted and unique. - -```jldoctest -julia> deleteat!([6, 5, 4, 3, 2, 1], 1:2:5) -3-element Array{Int64,1}: - 5 - 3 - 1 - -julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) -ERROR: ArgumentError: indices must be unique and sorted - in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:575 - ... -``` -""" -deleteat!(collection, itr) - """ read(stream::IO, T) @@ -4398,14 +4289,6 @@ after the end of the string. """ nextind -""" - symdiff(s1,s2...) - -Construct the symmetric difference of elements in the passed in sets or arrays. Maintains -order with arrays. -""" -symdiff - """ eta(x) @@ -4727,13 +4610,6 @@ Convert a hexadecimal string to the floating point number it represents. """ hex2num -""" - ndims(A) -> Integer - -Returns the number of dimensions of `A`. -""" -ndims - """ ishermitian(A) -> Bool @@ -5061,16 +4937,6 @@ Show a descriptive representation of an exception object. """ showerror -""" - setdiff(s1,s2) - -Construct the set of elements in `s1` but not `s2`. Maintains order with arrays. Note that -both arguments must be collections, and both will be iterated over. In particular, -`setdiff(set,element)` where `element` is a potential member of `set`, will not work in -general. -""" -setdiff - """ error(message::AbstractString) @@ -5883,28 +5749,6 @@ Return ``x^{1/3}``. The prefix operator `∛` is equivalent to `cbrt`. """ cbrt -""" - findprev(A, i) - -Find the previous index <= `i` of a non-zero element of `A`, or `0` if not found. -""" -findprev(A,i) - -""" - findprev(predicate, A, i) - -Find the previous index <= `i` of an element of `A` for which `predicate` returns `true`, or -`0` if not found. -""" -findprev(predicate::Function,A,i) - -""" - findprev(A, v, i) - -Find the previous index <= `i` of an element of `A` equal to `v` (using `==`), or `0` if not found. -""" -findprev(A,v,i) - """ matchall(r::Regex, s::AbstractString[, overlap::Bool=false]) -> Vector{AbstractString} @@ -6110,30 +5954,6 @@ all elements of the string. """ ispunct -""" - size(A, [dim...]) - -Returns a tuple containing the dimensions of `A`. Optionally you can specify the -dimension(s) you want the length of, and get the length of that dimension, or a tuple of the -lengths of dimensions you asked for. - - julia> A = rand(2,3,4); - - julia> size(A, 2) - 3 - - julia> size(A,3,2) - (4,3) -""" -size - -""" - findmin(A, dims) -> (minval, index) - -For an array input, returns the value and index of the minimum over the given dimensions. -""" -findmin(A,dims) - """ ismount(path) -> Bool @@ -6156,13 +5976,6 @@ Boolean not. """ Base.:(!) -""" - length(A) -> Integer - -Returns the number of elements in `A`. -""" -length(::AbstractArray) - """ length(collection) -> Integer @@ -6302,27 +6115,6 @@ returning a `Future` to the result. """ :@spawn -""" - findfirst(A) - -Return the index of the first non-zero value in `A` (determined by `A[i]!=0`). -""" -findfirst(A) - -""" - findfirst(A,v) - -Return the index of the first element equal to `v` in `A`. -""" -findfirst(A,v) - -""" - findfirst(predicate, A) - -Return the index of the first element of `A` for which `predicate` returns `true`. -""" -findfirst - """ promote_rule(type1, type2) @@ -6821,48 +6613,6 @@ false """ isbits -""" - findlast(A) - -Return the index of the last non-zero value in `A` (determined by `A[i]!=0`). -""" -findlast(A) - -""" - findlast(A, v) - -Return the index of the last element equal to `v` in `A`. -""" -findlast(A,v) - -""" - findlast(predicate, A) - -Return the index of the last element of `A` for which `predicate` returns `true`. -""" -findlast(::Function, A) - -""" - findnext(A, i) - -Find the next index >= `i` of a non-zero element of `A`, or `0` if not found. -""" -findnext - -""" - findnext(predicate, A, i) - -Find the next index >= `i` of an element of `A` for which `predicate` returns `true`, or `0` if not found. -""" -findnext(::Function,A,i) - -""" - findnext(A, v, i) - -Find the next index >= `i` of an element of `A` equal to `v` (using `==`), or `0` if not found. -""" -findnext(A,v,i) - """ angle(z) @@ -6909,18 +6659,6 @@ an array of the results `f(as...)` for each position. """ broadcast -""" - eigvecs(A, [eigvals,][permute=true,][scale=true]) -> Matrix - -Returns a matrix `M` whose columns are the eigenvectors of `A`. (The `k`th eigenvector can -be obtained from the slice `M[:, k]`.) The `permute` and `scale` keywords are the same as -for [`eigfact`](:func:`eigfact`). - -For [`SymTridiagonal`](:class:`SymTridiagonal`) matrices, if the optional vector of -eigenvalues `eigvals` is specified, returns the specific corresponding eigenvectors. -""" -eigvecs - """ ntoh(x) @@ -7364,38 +7102,6 @@ unsigned without checking for negative values. """ unsigned -""" - eigfact(A,[irange,][vl,][vu,][permute=true,][scale=true]) -> Eigen - -Computes the eigenvalue decomposition of `A`, returning an `Eigen` factorization object `F` -which contains the eigenvalues in `F[:values]` and the eigenvectors in the columns of the -matrix `F[:vectors]`. (The `k`th eigenvector can be obtained from the slice `F[:vectors][:, k]`.) - -The following functions are available for `Eigen` objects: `inv`, `det`. - -If `A` is [`Symmetric`](:class:`Symmetric`), [`Hermitian`](:class:`Hermitian`) or -[`SymTridiagonal`](:class:`SymTridiagonal`), it is possible to calculate only a subset of -the eigenvalues by specifying either a [`UnitRange`](:class:`UnitRange`) `irange` covering -indices of the sorted eigenvalues or a pair `vl` and `vu` for the lower and upper boundaries -of the eigenvalues. - -For general nonsymmetric matrices it is possible to specify how the matrix is balanced -before the eigenvector calculation. The option `permute=true` permutes the matrix to become -closer to upper triangular, and `scale=true` scales the matrix by its diagonal elements to -make rows and columns more equal in norm. The default is `true` for both options. -""" -eigfact(A,?,?,?,?) - -""" - eigfact(A, B) -> GeneralizedEigen - -Computes the generalized eigenvalue decomposition of `A` and `B`, returning a -`GeneralizedEigen` factorization object `F` which contains the generalized eigenvalues in -`F[:values]` and the generalized eigenvectors in the columns of the matrix `F[:vectors]`. -(The `k`th generalized eigenvector can be obtained from the slice `F[:vectors][:, k]`.) -""" -eigfact(A,B) - """ mkdir(path, [mode]) @@ -7404,7 +7110,6 @@ modified by the current file creation mask. """ mkdir - """ midpoints(e) @@ -7465,33 +7170,6 @@ Letter: Lowercase. """ islower -""" - eig(A,[irange,][vl,][vu,][permute=true,][scale=true]) -> D, V - -Computes eigenvalues and eigenvectors of `A`. See [`eigfact`](:func:`eigfact`) for details -on the `permute` and `scale` keyword arguments. The eigenvectors are returned columnwise. - -```jldoctest -julia> eig([1.0 0.0 0.0; 0.0 3.0 0.0; 0.0 0.0 18.0]) -([1.0,3.0,18.0], -[1.0 0.0 0.0; 0.0 1.0 0.0; 0.0 0.0 1.0]) -``` - -`eig` is a wrapper around [`eigfact`](:func:`eigfact`), extracting all parts of the -factorization to a tuple; where possible, using [`eigfact`](:func:`eigfact`) is recommended. -""" -eig(A,?,?,?) - -""" - eig(A, B) -> D, V - -Computes generalized eigenvalues and vectors of `A` with respect to `B`. - -`eig` is a wrapper around [`eigfact`](:func:`eigfact`), extracting all parts of the -factorization to a tuple; where possible, using [`eigfact`](:func:`eigfact`) is recommended. -""" -eig(A,B) - """ exp2(x) diff --git a/base/linalg/eigen.jl b/base/linalg/eigen.jl index 6e68c72d4b30f..93e9468fd24a7 100644 --- a/base/linalg/eigen.jl +++ b/base/linalg/eigen.jl @@ -54,6 +54,27 @@ function eigfact!{T<:BlasComplex}(A::StridedMatrix{T}; permute::Bool=true, scale ishermitian(A) && return eigfact!(Hermitian(A)) return Eigen(LAPACK.geevx!(permute ? (scale ? 'B' : 'P') : (scale ? 'S' : 'N'), 'N', 'V', 'N', A)[[2,4]]...) end + +""" + eigfact(A,[irange,][vl,][vu,][permute=true,][scale=true]) -> Eigen + +Computes the eigenvalue decomposition of `A`, returning an [`Eigen`](:obj:`Eigen`) factorization object `F` +which contains the eigenvalues in `F[:values]` and the eigenvectors in the columns of the +matrix `F[:vectors]`. (The `k`th eigenvector can be obtained from the slice `F[:vectors][:, k]`.) + +The following functions are available for `Eigen` objects: [`inv`](:func:`inv`), [`det`](:func:`det`), and [`isposdef`](:func:`isposdef`). + +If `A` is [`Symmetric`](:class:`Symmetric`), [`Hermitian`](:class:`Hermitian`) or +[`SymTridiagonal`](:class:`SymTridiagonal`), it is possible to calculate only a subset of +the eigenvalues by specifying either a [`UnitRange`](:class:`UnitRange`) `irange` covering +indices of the sorted eigenvalues or a pair `vl` and `vu` for the lower and upper boundaries +of the eigenvalues. + +For general nonsymmetric matrices it is possible to specify how the matrix is balanced +before the eigenvector calculation. The option `permute=true` permutes the matrix to become +closer to upper triangular, and `scale=true` scales the matrix by its diagonal elements to +make rows and columns more equal in norm. The default is `true` for both options. +""" function eigfact{T}(A::StridedMatrix{T}; permute::Bool=true, scale::Bool=true) S = promote_type(Float32, typeof(one(T)/norm(one(T)))) eigfact!(copy_oftype(A, S), permute = permute, scale = scale) @@ -64,12 +85,40 @@ function eig(A::Union{Number, StridedMatrix}; permute::Bool=true, scale::Bool=tr F = eigfact(A, permute=permute, scale=scale) F.values, F.vectors end + +""" + eig(A,[irange,][vl,][vu,][permute=true,][scale=true]) -> D, V + +Computes eigenvalues (`D`) and eigenvectors (`V`) of `A`. +See [`eigfact`](:func:`eigfact`) for details on the +`irange`, `vl`, and `vu` arguments +and the `permute` and `scale` keyword arguments. +The eigenvectors are returned columnwise. + +```jldoctest +julia> eig([1.0 0.0 0.0; 0.0 3.0 0.0; 0.0 0.0 18.0]) +([1.0,3.0,18.0], +[1.0 0.0 0.0; 0.0 1.0 0.0; 0.0 0.0 1.0]) +``` + +`eig` is a wrapper around [`eigfact`](:func:`eigfact`), extracting all parts of the +factorization to a tuple; where possible, using [`eigfact`](:func:`eigfact`) is recommended. +""" function eig(A::AbstractMatrix, args...) F = eigfact(A, args...) F.values, F.vectors end -#Calculates eigenvectors +""" + eigvecs(A, [eigvals,][permute=true,][scale=true]) -> Matrix + +Returns a matrix `M` whose columns are the eigenvectors of `A`. (The `k`th eigenvector can +be obtained from the slice `M[:, k]`.) The `permute` and `scale` keywords are the same as +for [`eigfact`](:func:`eigfact`). + +For [`SymTridiagonal`](:class:`SymTridiagonal`) matrices, if the optional vector of +eigenvalues `eigvals` is specified, returns the specific corresponding eigenvectors. +""" eigvecs(A::Union{Number, AbstractMatrix}; permute::Bool=true, scale::Bool=true) = eigvecs(eigfact(A, permute=permute, scale=scale)) eigvecs{T,V,S,U}(F::Union{Eigen{T,V,S,U}, GeneralizedEigen{T,V,S,U}}) = F[:vectors]::S @@ -77,10 +126,9 @@ eigvecs{T,V,S,U}(F::Union{Eigen{T,V,S,U}, GeneralizedEigen{T,V,S,U}}) = F[:vecto eigvals{T,V,S,U}(F::Union{Eigen{T,V,S,U}, GeneralizedEigen{T,V,S,U}}) = F[:values]::U """ - eigvals!(A,[irange,][vl,][vu]) -> values -Same as `eigvals`, but saves space by overwriting the input `A` (and `B`), instead of creating a copy. +Same as [`eigvals`](:func:`eigvals`), but saves space by overwriting the input `A`, instead of creating a copy. """ function eigvals!{T<:BlasReal}(A::StridedMatrix{T}; permute::Bool=true, scale::Bool=true) issymmetric(A) && return eigvals!(Symmetric(A)) @@ -100,8 +148,37 @@ function eigvals{T<:Number}(x::T; kwargs...) return imag(val) == 0 ? [real(val)] : [val] end -# TO DO: Put message about not being able to sort complex numbers back in! -#Computes maximum and minimum eigenvalue +""" + eigmax(A; permute::Bool=true, scale::Bool=true) + +Returns the largest eigenvalue of `A`. +The option `permute=true` permutes the matrix to become +closer to upper triangular, and `scale=true` scales the matrix by its diagonal elements to +make rows and columns more equal in norm. +Note that if the eigenvalues of `A` are complex, +this method will fail, since complex numbers cannot +be sorted. + +```jldoctest +julia> A = [0 im; -im 0] +2×2 Array{Complex{Int64},2}: + 0+0im 0+1im + 0-1im 0+0im + +julia> eigmax(A) +1.0 + +julia> A = [0 im; -1 0] +2×2 Array{Complex{Int64},2}: + 0+0im 0+1im + -1+0im 0+0im + +julia> eigmax(A) +ERROR: DomainError: + in #eigmax#30(::Bool, ::Bool, ::Function, ::Array{Complex{Int64},2}) at ./linalg/eigen.jl:108 + in eigmax(::Array{Complex{Int64},2}) at ./linalg/eigen.jl:106 +``` +""" function eigmax(A::Union{Number, StridedMatrix}; permute::Bool=true, scale::Bool=true) v = eigvals(A, permute = permute, scale = scale) if eltype(v)<:Complex @@ -109,6 +186,38 @@ function eigmax(A::Union{Number, StridedMatrix}; permute::Bool=true, scale::Bool end maximum(v) end + +""" + eigmin(A; permute::Bool=true, scale::Bool=true) + +Returns the smallest eigenvalue of `A`. +The option `permute=true` permutes the matrix to become +closer to upper triangular, and `scale=true` scales the matrix by its diagonal elements to +make rows and columns more equal in norm. +Note that if the eigenvalues of `A` are complex, +this method will fail, since complex numbers cannot +be sorted. + +```jldoctest +julia> A = [0 im; -im 0] +2×2 Array{Complex{Int64},2}: + 0+0im 0+1im + 0-1im 0+0im + +julia> eigmin(A) +-1.0 + +julia> A = [0 im; -1 0] +2×2 Array{Complex{Int64},2}: + 0+0im 0+1im + -1+0im 0+0im + +julia> eigmin(A) +ERROR: DomainError: + in #eigmin#31(::Bool, ::Bool, ::Function, ::Array{Complex{Int64},2}) at ./linalg/eigen.jl:115 + in eigmin(::Array{Complex{Int64},2}) at ./linalg/eigen.jl:113 +``` +""" function eigmin(A::Union{Number, StridedMatrix}; permute::Bool=true, scale::Bool=true) v = eigvals(A, permute = permute, scale = scale) if eltype(v)<:Complex @@ -149,6 +258,15 @@ function eigfact!{T<:BlasComplex}(A::StridedMatrix{T}, B::StridedMatrix{T}) alpha, beta, _, vr = LAPACK.ggev!('N', 'V', A, B) return GeneralizedEigen(alpha./beta, vr) end + +""" + eigfact(A, B) -> GeneralizedEigen + +Computes the generalized eigenvalue decomposition of `A` and `B`, returning a +`GeneralizedEigen` factorization object `F` which contains the generalized eigenvalues in +`F[:values]` and the generalized eigenvectors in the columns of the matrix `F[:vectors]`. +(The `k`th generalized eigenvector can be obtained from the slice `F[:vectors][:, k]`.) +""" function eigfact{TA,TB}(A::AbstractMatrix{TA}, B::AbstractMatrix{TB}) S = promote_type(Float32, typeof(one(TA)/norm(one(TA))),TB) return eigfact!(copy_oftype(A, S), copy_oftype(B, S)) @@ -156,6 +274,30 @@ end eigfact(A::Number, B::Number) = eigfact(fill(A,1,1), fill(B,1,1)) +""" + eig(A, B) -> D, V + +Computes generalized eigenvalues (`D`) and vectors (`V`) of `A` with respect to `B`. + +`eig` is a wrapper around [`eigfact`](:func:`eigfact`), extracting all parts of the +factorization to a tuple; where possible, using [`eigfact`](:func:`eigfact`) is recommended. + +```jldoctest +julia> A = [1 0; 0 -1] +2×2 Array{Int64,2}: + 1 0 + 0 -1 + +julia> B = [0 1; 1 0] +2×2 Array{Int64,2}: + 0 1 + 1 0 + +julia> eig(A, B) +(Complex{Float64}[0.0+1.0im,0.0-1.0im], +Complex{Float64}[0.0-1.0im 0.0+1.0im; -1.0-0.0im -1.0+0.0im]) +``` +""" function eig(A::AbstractMatrix, B::AbstractMatrix) F = eigfact(A,B) F.values, F.vectors @@ -165,6 +307,11 @@ function eig(A::Number, B::Number) F.values, F.vectors end +""" + eigvals!(A, B) -> values + +Same as [`eigvals`](:func:`eigvals`), but saves space by overwriting the input `A` (and `B`), instead of creating copies. +""" function eigvals!{T<:BlasReal}(A::StridedMatrix{T}, B::StridedMatrix{T}) issymmetric(A) && isposdef(B) && return eigvals!(Symmetric(A), Symmetric(B)) alphar, alphai, beta, vl, vr = LAPACK.ggev!('N', 'N', A, B) @@ -175,11 +322,57 @@ function eigvals!{T<:BlasComplex}(A::StridedMatrix{T}, B::StridedMatrix{T}) alpha, beta, vl, vr = LAPACK.ggev!('N', 'N', A, B) alpha./beta end + +""" + eigvals(A, B) -> values + +Computes the generalized eigenvalues of `A` and `B`. + +```jldoctest +julia> A = [1 0; 0 -1] +2×2 Array{Int64,2}: + 1 0 + 0 -1 + +julia> B = [0 1; 1 0] +2×2 Array{Int64,2}: + 0 1 + 1 0 + +julia> eigvals(A,B) +2-element Array{Complex{Float64},1}: + 0.0+1.0im + 0.0-1.0im +``` +""" function eigvals{TA,TB}(A::AbstractMatrix{TA}, B::AbstractMatrix{TB}) S = promote_type(Float32, typeof(one(TA)/norm(one(TA))),TB) return eigvals!(copy_oftype(A, S), copy_oftype(B, S)) end +""" + eigvecs(A, B) -> Matrix + +Returns a matrix `M` whose columns are the generalized eigenvectors of `A` and `B`. (The `k`th eigenvector can +be obtained from the slice `M[:, k]`.) + +```jldoctest +julia> A = [1 0; 0 -1] +2×2 Array{Int64,2}: + 1 0 + 0 -1 + +julia> B = [0 1; 1 0] +2×2 Array{Int64,2}: + 0 1 + 1 0 + +julia> eigvecs(A, B) +2×2 Array{Complex{Float64},2}: + 0.0-1.0im 0.0+1.0im + -1.0-0.0im -1.0+0.0im +``` +""" eigvecs(A::AbstractMatrix, B::AbstractMatrix) = eigvecs(eigfact(A, B)) # Conversion methods @@ -189,4 +382,4 @@ convert(::Type{AbstractMatrix}, F::Eigen) = F.vectors * Diagonal(F.values) / F.v convert(::Type{AbstractArray}, F::Eigen) = convert(AbstractMatrix, F) convert(::Type{Matrix}, F::Eigen) = convert(Array, convert(AbstractArray, F)) convert(::Type{Array}, F::Eigen) = convert(Matrix, F) -full(F::Eigen) = convert(Array, F) \ No newline at end of file +full(F::Eigen) = convert(Array, F) diff --git a/base/reducedim.jl b/base/reducedim.jl index 047cc9af80e92..95770ea0abafd 100644 --- a/base/reducedim.jl +++ b/base/reducedim.jl @@ -380,6 +380,11 @@ function findmin!{R}(rval::AbstractArray{R}, findminmax!(<, initarray!(rval, scalarmin, init), rind, A) end +""" + findmin(A, region) -> (minval, index) + +For an array input, returns the value and index of the minimum over the given region. +""" function findmin{T}(A::AbstractArray{T}, region) if isempty(A) return (similar(A, reduced_dims0(A, region)), @@ -402,6 +407,11 @@ function findmax!{R}(rval::AbstractArray{R}, findminmax!(>, initarray!(rval, scalarmax, init), rind, A) end +""" + findmax(A, region) -> (maxval, index) + +For an array input, returns the value and index of the maximum over the given region. +""" function findmax{T}(A::AbstractArray{T}, region) if isempty(A) return (similar(A, reduced_dims0(A,region)), diff --git a/doc/stdlib/arrays.rst b/doc/stdlib/arrays.rst index 78749eac4ed17..56794fe75b417 100644 --- a/doc/stdlib/arrays.rst +++ b/doc/stdlib/arrays.rst @@ -9,21 +9,28 @@ Basic functions --------------- -.. function:: ndims(A) -> Integer +.. function:: ndims(A::AbstractArray) -> Integer .. Docstring generated from Julia source Returns the number of dimensions of ``A``\ . -.. function:: size(A, [dim...]) + .. doctest:: + + julia> A = ones(3,4,5); + + julia> ndims(A) + 3 + +.. function:: size(A::AbstractArray, [dim...]) .. Docstring generated from Julia source Returns a tuple containing the dimensions of ``A``\ . Optionally you can specify the dimension(s) you want the length of, and get the length of that dimension, or a tuple of the lengths of dimensions you asked for. - .. code-block:: julia + .. doctest:: - julia> A = rand(2,3,4); + julia> A = ones(2,3,4); julia> size(A, 2) 3 @@ -43,12 +50,19 @@ Basic functions Returns the valid range of indices for array ``A`` along dimension ``d``\ . -.. function:: length(A) -> Integer +.. function:: length(A::AbstractArray) -> Integer .. Docstring generated from Julia source Returns the number of elements in ``A``\ . + .. doctest:: + + julia> A = ones(3,4,5); + + julia> length(A) + 60 + .. function:: eachindex(A...) .. Docstring generated from Julia source @@ -116,18 +130,35 @@ Basic functions Convert an array to its complex conjugate in-place. -.. function:: stride(A, k) +.. function:: stride(A, k::Integer) .. Docstring generated from Julia source Returns the distance in memory (in number of elements) between adjacent elements in dimension ``k``\ . + .. doctest:: + + julia> A = ones(3,4,5); + + julia> stride(A,2) + 3 + + julia> stride(A,3) + 12 + .. function:: strides(A) .. Docstring generated from Julia source Returns a tuple of the memory strides in each dimension. + .. doctest:: + + julia> A = ones(3,4,5); + + julia> strides(A) + (1,3,12) + .. function:: ind2sub(dims, index) -> subscripts .. Docstring generated from Julia source @@ -608,19 +639,62 @@ Indexing, Assignment, and Concatenation .. Docstring generated from Julia source - Return a vector of the linear indexes of the non-zeros in ``A`` (determined by ``A[i]!=0``\ ). A common use of this is to convert a boolean array to an array of indexes of the ``true`` elements. + Return a vector of the linear indexes of the non-zeros in ``A`` (determined by ``A[i]!=0``\ ). A common use of this is to convert a boolean array to an array of indexes of the ``true`` elements. If there are no non-zero elements of ``A``\ , ``find`` returns an empty array. -.. function:: find(f,A) + .. doctest:: + + julia> A = [true false; false true] + 2×2 Array{Bool,2}: + true false + false true + + julia> find(A) + 2-element Array{Int64,1}: + 1 + 4 + +.. function:: find(f::Function, A) .. Docstring generated from Julia source - Return a vector of the linear indexes of ``A`` where ``f`` returns ``true``\ . + Return a vector ``I`` of the linear indexes of ``A`` where ``f(A[I])`` returns ``true``\ . If there are no such elements of ``A``\ , find returns an empty array. + + .. doctest:: + + julia> A = [1 2; 3 4] + 2×2 Array{Int64,2}: + 1 2 + 3 4 + + julia> find(isodd,A) + 2-element Array{Int64,1}: + 1 + 2 .. function:: findn(A) .. Docstring generated from Julia source - Return a vector of indexes for each dimension giving the locations of the non-zeros in ``A`` (determined by ``A[i]!=0``\ ). + Return a vector of indexes for each dimension giving the locations of the non-zeros in ``A`` (determined by ``A[i]!=0``\ ). If there are no non-zero elements of ``A``\ , ``findn`` returns a 2-tuple of empty arrays. + + .. doctest:: + + julia> A = [1 2 0; 0 0 3; 0 4 0] + 3×3 Array{Int64,2}: + 1 2 0 + 0 0 3 + 0 4 0 + + julia> findn(A) + ([1,1,3,2],[1,2,2,3]) + + julia> A = zeros(2,2) + 2×2 Array{Float64,2}: + 0.0 0.0 + 0.0 0.0 + + julia> findn(A) + (Int64[],Int64[]) .. function:: findnz(A) @@ -628,77 +702,249 @@ Indexing, Assignment, and Concatenation Return a tuple ``(I, J, V)`` where ``I`` and ``J`` are the row and column indexes of the non-zero values in matrix ``A``\ , and ``V`` is a vector of the non-zero values. + .. doctest:: + + julia> A = [1 2 0; 0 0 3; 0 4 0] + 3×3 Array{Int64,2}: + 1 2 0 + 0 0 3 + 0 4 0 + + julia> findnz(A) + ([1,1,3,2],[1,2,2,3],[1,2,4,3]) + .. function:: findfirst(A) .. Docstring generated from Julia source - Return the index of the first non-zero value in ``A`` (determined by ``A[i]!=0``\ ). + Return the linear index of the first non-zero value in ``A`` (determined by ``A[i]!=0``\ ). Returns ``0`` if no such value is found. + + .. doctest:: -.. function:: findfirst(A,v) + julia> A = [0 0; 1 0] + 2×2 Array{Int64,2}: + 0 0 + 1 0 + + julia> findfirst(A) + 2 + +.. function:: findfirst(A, v) .. Docstring generated from Julia source - Return the index of the first element equal to ``v`` in ``A``\ . + Return the linear index of the first element equal to ``v`` in ``A``\ . Returns ``0`` if ``v`` is not found. + + .. doctest:: + + julia> A = [4 6; 2 2] + 2×2 Array{Int64,2}: + 4 6 + 2 2 + + julia> findfirst(A,2) + 2 + + julia> findfirst(A,3) + 0 -.. function:: findfirst(predicate, A) +.. function:: findfirst(predicate::Function, A) .. Docstring generated from Julia source - Return the index of the first element of ``A`` for which ``predicate`` returns ``true``\ . + Return the linear index of the first element of ``A`` for which ``predicate`` returns ``true``\ . Returns ``0`` if there is no such element. + + .. doctest:: + + julia> A = [1 4; 2 2] + 2×2 Array{Int64,2}: + 1 4 + 2 2 + + julia> findfirst(iseven, A) + 2 + + julia> findfirst(x -> x>10, A) + 0 .. function:: findlast(A) .. Docstring generated from Julia source - Return the index of the last non-zero value in ``A`` (determined by ``A[i]!=0``\ ). + Return the linear index of the last non-zero value in ``A`` (determined by ``A[i]!=0``\ ). Returns ``0`` if there is no non-zero value in ``A``\ . + + .. doctest:: + + julia> A = [1 0; 1 0] + 2×2 Array{Int64,2}: + 1 0 + 1 0 + + julia> findlast(A) + 2 + + julia> A = zeros(2,2) + 2×2 Array{Float64,2}: + 0.0 0.0 + 0.0 0.0 + + julia> findlast(A) + 0 .. function:: findlast(A, v) .. Docstring generated from Julia source - Return the index of the last element equal to ``v`` in ``A``\ . + Return the linear index of the last element equal to ``v`` in ``A``\ . Returns ``0`` if there is no element of ``A`` equal to ``v``\ . -.. function:: findlast(predicate, A) + .. doctest:: + + julia> A = [1 2; 2 1] + 2×2 Array{Int64,2}: + 1 2 + 2 1 + + julia> findlast(A,1) + 4 + + julia> findlast(A,2) + 3 + + julia> findlast(A,3) + 0 + +.. function:: findlast(predicate::Function, A) .. Docstring generated from Julia source - Return the index of the last element of ``A`` for which ``predicate`` returns ``true``\ . + Return the linear index of the last element of ``A`` for which ``predicate`` returns ``true``\ . Returns ``0`` if there is no such element. -.. function:: findnext(A, i) + .. doctest:: + + julia> A = [1 2; 3 4] + 2×2 Array{Int64,2}: + 1 2 + 3 4 + + julia> findlast(isodd, A) + 2 + + julia> findlast(x -> x > 5, A) + 0 + +.. function:: findnext(A, i::Integer) .. Docstring generated from Julia source - Find the next index >= ``i`` of a non-zero element of ``A``\ , or ``0`` if not found. + Find the next linear index >= ``i`` of a non-zero element of ``A``\ , or ``0`` if not found. + + .. doctest:: + + julia> A = [0 0; 1 0] + 2×2 Array{Int64,2}: + 0 0 + 1 0 + + julia> findnext(A,1) + 2 -.. function:: findnext(predicate, A, i) + julia> findnext(A,3) + 0 + +.. function:: findnext(predicate::Function, A, i::Integer) .. Docstring generated from Julia source - Find the next index >= ``i`` of an element of ``A`` for which ``predicate`` returns ``true``\ , or ``0`` if not found. + Find the next linear index >= ``i`` of an element of ``A`` for which ``predicate`` returns ``true``\ , or ``0`` if not found. + + .. doctest:: + + julia> A = [1 4; 2 2] + 2×2 Array{Int64,2}: + 1 4 + 2 2 + + julia> findnext(isodd, A, 1) + 1 -.. function:: findnext(A, v, i) + julia> findnext(isodd, A, 2) + 0 + +.. function:: findnext(A, v, i::Integer) .. Docstring generated from Julia source - Find the next index >= ``i`` of an element of ``A`` equal to ``v`` (using ``==``\ ), or ``0`` if not found. + Find the next linear index >= ``i`` of an element of ``A`` equal to ``v`` (using ``==``\ ), or ``0`` if not found. + + .. doctest:: + + julia> A = [1 4; 2 2] + 2×2 Array{Int64,2}: + 1 4 + 2 2 -.. function:: findprev(A, i) + julia> findnext(A,4,4) + 0 + + julia> findnext(A,4,3) + 3 + +.. function:: findprev(A, i::Integer) .. Docstring generated from Julia source - Find the previous index <= ``i`` of a non-zero element of ``A``\ , or ``0`` if not found. + Find the previous linear index <= ``i`` of a non-zero element of ``A``\ , or ``0`` if not found. + + .. doctest:: + + julia> A = [0 0; 1 2] + 2×2 Array{Int64,2}: + 0 0 + 1 2 -.. function:: findprev(predicate, A, i) + julia> findprev(A,2) + 2 + + julia> findprev(A,1) + 0 + +.. function:: findprev(predicate::Function, A, i::Integer) .. Docstring generated from Julia source - Find the previous index <= ``i`` of an element of ``A`` for which ``predicate`` returns ``true``\ , or ``0`` if not found. + Find the previous linear index <= ``i`` of an element of ``A`` for which ``predicate`` returns ``true``\ , or ``0`` if not found. + + .. doctest:: + + julia> A = [4 6; 1 2] + 2×2 Array{Int64,2}: + 4 6 + 1 2 + + julia> findprev(isodd, A, 1) + 0 -.. function:: findprev(A, v, i) + julia> findprev(isodd, A, 3) + 2 + +.. function:: findprev(A, v, i::Integer) .. Docstring generated from Julia source - Find the previous index <= ``i`` of an element of ``A`` equal to ``v`` (using ``==``\ ), or ``0`` if not found. + Find the previous linear index <= ``i`` of an element of ``A`` equal to ``v`` (using ``==``\ ), or ``0`` if not found. + + .. doctest:: + + julia> A = [0 0; 1 2] + 2×2 Array{Int64,2}: + 0 0 + 1 2 + + julia> findprev(A, 1, 4) + 2 + + julia> findprev(A, 1, 1) + 0 .. function:: permutedims(A, perm) diff --git a/doc/stdlib/collections.rst b/doc/stdlib/collections.rst index 7d3fba881b555..a8b4285b754c4 100644 --- a/doc/stdlib/collections.rst +++ b/doc/stdlib/collections.rst @@ -480,7 +480,7 @@ Iterable Collections .. Docstring generated from Julia source - Returns the index of the maximum element in a collection. + Returns the index of the maximum element in a collection. The collection must not be empty. .. doctest:: @@ -491,7 +491,7 @@ Iterable Collections .. Docstring generated from Julia source - Returns the index of the minimum element in a collection. + Returns the index of the minimum element in a collection. The collection must not be empty. .. doctest:: @@ -509,11 +509,11 @@ Iterable Collections julia> findmax([8,0.1,-9,pi]) (8.0,1) -.. function:: findmax(A, dims) -> (maxval, index) +.. function:: findmax(A, region) -> (maxval, index) .. Docstring generated from Julia source - For an array input, returns the value and index of the maximum over the given dimensions. + For an array input, returns the value and index of the maximum over the given region. .. function:: findmin(itr) -> (x, index) @@ -526,11 +526,11 @@ Iterable Collections julia> findmin([8,0.1,-9,pi]) (-9.0,3) -.. function:: findmin(A, dims) -> (minval, index) +.. function:: findmin(A, region) -> (minval, index) .. Docstring generated from Julia source - For an array input, returns the value and index of the minimum over the given dimensions. + For an array input, returns the value and index of the minimum over the given region. .. function:: findmax!(rval, rind, A, [init=true]) -> (maxval, index) @@ -1162,11 +1162,18 @@ Set-Like Collections Construct the intersection of two or more sets. Maintains order and multiplicity of the first argument for arrays and ranges. -.. function:: setdiff(s1,s2) +.. function:: setdiff(a, b) .. Docstring generated from Julia source - Construct the set of elements in ``s1`` but not ``s2``\ . Maintains order with arrays. Note that both arguments must be collections, and both will be iterated over. In particular, ``setdiff(set,element)`` where ``element`` is a potential member of ``set``\ , will not work in general. + Construct the set of elements in ``a`` but not ``b``\ . Maintains order with arrays. Note that both arguments must be collections, and both will be iterated over. In particular, ``setdiff(set,element)`` where ``element`` is a potential member of ``set``\ , will not work in general. + + .. doctest:: + + julia> setdiff([1,2,3],[3,4,5]) + 2-element Array{Int64,1}: + 1 + 2 .. function:: setdiff!(s, iterable) @@ -1174,12 +1181,20 @@ Set-Like Collections Remove each element of ``iterable`` from set ``s`` in-place. -.. function:: symdiff(s1,s2...) +.. function:: symdiff(a, b, rest...) .. Docstring generated from Julia source Construct the symmetric difference of elements in the passed in sets or arrays. Maintains order with arrays. + .. doctest:: + + julia> symdiff([1,2,3],[3,4,5],[4,5,6]) + 3-element Array{Int64,1}: + 1 + 2 + 6 + .. function:: symdiff!(s, n) .. Docstring generated from Julia source @@ -1332,11 +1347,11 @@ Dequeues 2 1 -.. function:: deleteat!(collection, index) +.. function:: deleteat!(a::Vector, i::Integer) .. Docstring generated from Julia source - Remove the item at the given ``index`` and return the modified ``collection``\ . Subsequent items are shifted to fill the resulting gap. + Remove the item at the given ``i`` and return the modified ``a``\ . Subsequent items are shifted to fill the resulting gap. .. doctest:: @@ -1348,11 +1363,11 @@ Dequeues 2 1 -.. function:: deleteat!(collection, itr) +.. function:: deleteat!(a::Vector, inds) .. Docstring generated from Julia source - Remove the items at the indices given by ``itr``\ , and return the modified ``collection``\ . Subsequent items are shifted to fill the resulting gap. ``itr`` must be sorted and unique. + Remove the items at the indices given by ``inds``\ , and return the modified ``a``\ . Subsequent items are shifted to fill the resulting gap. ``inds`` must be sorted and unique. .. doctest:: @@ -1364,7 +1379,7 @@ Dequeues julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) ERROR: ArgumentError: indices must be unique and sorted - in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:575 + in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:576 ... .. function:: splice!(collection, index, [replacement]) -> item @@ -1527,18 +1542,57 @@ changed efficiently. A ``PriorityQueue`` acts like a ``Dict``\ , mapping values to their priorities, with the addition of a ``dequeue!`` function to remove the lowest priority element. + .. doctest:: + + julia> a = Base.Collections.PriorityQueue(["a","b","c"],[2,3,1],Base.Order.Forward) + Base.Collections.PriorityQueue{String,Int64,Base.Order.ForwardOrdering} with 3 entries: + "c" => 1 + "b" => 3 + "a" => 2 + .. function:: enqueue!(pq, k, v) .. Docstring generated from Julia source Insert the a key ``k`` into a priority queue ``pq`` with priority ``v``\ . + .. doctest:: + + julia> a = Base.Collections.PriorityQueue(["a","b","c"],[2,3,1],Base.Order.Forward) + Base.Collections.PriorityQueue{String,Int64,Base.Order.ForwardOrdering} with 3 entries: + "c" => 1 + "b" => 3 + "a" => 2 + + julia> Base.Collections.enqueue!(a, "d", 4) + Base.Collections.PriorityQueue{String,Int64,Base.Order.ForwardOrdering} with 4 entries: + "c" => 1 + "b" => 3 + "a" => 2 + "d" => 4 + .. function:: dequeue!(pq) .. Docstring generated from Julia source Remove and return the lowest priority key from a priority queue. + .. doctest:: + + julia> a = Base.Collections.PriorityQueue(["a","b","c"],[2,3,1],Base.Order.Forward) + Base.Collections.PriorityQueue{String,Int64,Base.Order.ForwardOrdering} with 3 entries: + "c" => 1 + "b" => 3 + "a" => 2 + + julia> Base.Collections.dequeue!(a) + "c" + + julia> a + Base.Collections.PriorityQueue{String,Int64,Base.Order.ForwardOrdering} with 2 entries: + "b" => 3 + "a" => 2 + .. function:: peek(pq) .. Docstring generated from Julia source @@ -1575,24 +1629,58 @@ lower level functions for performing binary heap operations on arrays. Each function takes an optional ordering argument. If not given, default ordering is used, so that elements popped from the heap are given in ascending order. -.. function:: heapify(v, [ord]) +.. function:: heapify(v, ord::Ordering=Forward) .. Docstring generated from Julia source Returns a new vector in binary heap order, optionally using the given ordering. -.. function:: heapify!(v, [ord]) + .. doctest:: + + julia> a = [1,3,4,5,2]; + + julia> Base.Collections.heapify(a) + 5-element Array{Int64,1}: + 1 + 2 + 4 + 5 + 3 + + julia> Base.Collections.heapify(a, Base.Order.Reverse) + 5-element Array{Int64,1}: + 5 + 3 + 4 + 1 + 2 + +.. function:: heapify!(v, ord::Ordering=Forward) .. Docstring generated from Julia source In-place :func:`heapify`\ . -.. function:: isheap(v, [ord]) +.. function:: isheap(v, ord::Ordering=Forward) .. Docstring generated from Julia source Return ``true`` if an array is heap-ordered according to the given order. + .. doctest:: + + julia> a = [1,2,3] + 3-element Array{Int64,1}: + 1 + 2 + 3 + + julia> Base.Collections.isheap(a,Base.Order.Forward) + true + + julia> Base.Collections.isheap(a,Base.Order.Reverse) + false + .. function:: heappush!(v, x, [ord]) .. Docstring generated from Julia source diff --git a/doc/stdlib/io-network.rst b/doc/stdlib/io-network.rst index 6f4737b133498..4e747354dda2a 100644 --- a/doc/stdlib/io-network.rst +++ b/doc/stdlib/io-network.rst @@ -624,11 +624,11 @@ Text I/O The columns are assumed to be separated by one or more whitespaces. The end of line delimiter is taken as ``\n``\ . If all data is numeric, the result will be a numeric array. If some elements cannot be parsed as numbers, a heterogeneous array of numbers and strings is returned. -.. function:: writedlm(f, A, dl='\\t'; opts) +.. function:: writedlm(f, A, delim='\\t'; opts) .. Docstring generated from Julia source - Write ``A`` (a vector, matrix or an iterable collection of iterable rows) as text to ``f`` (either a filename string or an ``IO`` stream) using the given delimiter ``delim`` (which defaults to tab, but can be any printable Julia object, typically a ``Char`` or ``AbstractString``\ ). + Write ``A`` (a vector, matrix, or an iterable collection of iterable rows) as text to ``f`` (either a filename string or an :class:`IO` stream) using the given delimiter ``delim`` (which defaults to tab, but can be any printable Julia object, typically a ``Char`` or ``AbstractString``\ ). For example, two vectors ``x`` and ``y`` of the same length can be written as two columns of tab-delimited text to ``f`` by either ``writedlm(f, [x y])`` or by ``writedlm(f, zip(x, y))``\ . @@ -642,7 +642,7 @@ Text I/O .. Docstring generated from Julia source - Equivalent to ``writedlm`` with ``delim`` set to comma. + Equivalent to :func:`writedlm` with ``delim`` set to comma. .. function:: Base64EncodePipe(ostream) diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index 8352695c8b3c5..2bd41341c74fb 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -580,7 +580,7 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Computes eigenvalues and eigenvectors of ``A``\ . See :func:`eigfact` for details on the ``permute`` and ``scale`` keyword arguments. The eigenvectors are returned columnwise. + Computes eigenvalues (``D``\ ) and eigenvectors (``V``\ ) of ``A``\ . See :func:`eigfact` for details on the ``irange``\ , ``vl``\ , and ``vu`` arguments and the ``permute`` and ``scale`` keyword arguments. The eigenvectors are returned columnwise. .. doctest:: @@ -594,10 +594,26 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Computes generalized eigenvalues and vectors of ``A`` with respect to ``B``\ . + Computes generalized eigenvalues (``D``\ ) and vectors (``V``\ ) of ``A`` with respect to ``B``\ . ``eig`` is a wrapper around :func:`eigfact`\ , extracting all parts of the factorization to a tuple; where possible, using :func:`eigfact` is recommended. + .. doctest:: + + julia> A = [1 0; 0 -1] + 2×2 Array{Int64,2}: + 1 0 + 0 -1 + + julia> B = [0 1; 1 0] + 2×2 Array{Int64,2}: + 0 1 + 1 0 + + julia> eig(A, B) + (Complex{Float64}[0.0+1.0im,0.0-1.0im], + Complex{Float64}[0.0-1.0im 0.0+1.0im; -1.0-0.0im -1.0+0.0im]) + .. function:: eigvals(A,[irange,][vl,][vu]) -> values .. Docstring generated from Julia source @@ -610,19 +626,59 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Same as ``eigvals``\ , but saves space by overwriting the input ``A`` (and ``B``\ ), instead of creating a copy. + Same as :func:`eigvals`\ , but saves space by overwriting the input ``A``\ , instead of creating a copy. -.. function:: eigmax(A) +.. function:: eigmax(A; permute::Bool=true, scale::Bool=true) .. Docstring generated from Julia source - Returns the largest eigenvalue of ``A``\ . + Returns the largest eigenvalue of ``A``\ . The option ``permute=true`` permutes the matrix to become closer to upper triangular, and ``scale=true`` scales the matrix by its diagonal elements to make rows and columns more equal in norm. Note that if the eigenvalues of ``A`` are complex, this method will fail, since complex numbers cannot be sorted. + + .. doctest:: + + julia> A = [0 im; -im 0] + 2×2 Array{Complex{Int64},2}: + 0+0im 0+1im + 0-1im 0+0im + + julia> eigmax(A) + 1.0 -.. function:: eigmin(A) + julia> A = [0 im; -1 0] + 2×2 Array{Complex{Int64},2}: + 0+0im 0+1im + -1+0im 0+0im + + julia> eigmax(A) + ERROR: DomainError: + in #eigmax#30(::Bool, ::Bool, ::Function, ::Array{Complex{Int64},2}) at ./linalg/eigen.jl:108 + in eigmax(::Array{Complex{Int64},2}) at ./linalg/eigen.jl:106 + +.. function:: eigmin(A; permute::Bool=true, scale::Bool=true) .. Docstring generated from Julia source - Returns the smallest eigenvalue of ``A``\ . + Returns the smallest eigenvalue of ``A``\ . The option ``permute=true`` permutes the matrix to become closer to upper triangular, and ``scale=true`` scales the matrix by its diagonal elements to make rows and columns more equal in norm. Note that if the eigenvalues of ``A`` are complex, this method will fail, since complex numbers cannot be sorted. + + .. doctest:: + + julia> A = [0 im; -im 0] + 2×2 Array{Complex{Int64},2}: + 0+0im 0+1im + 0-1im 0+0im + + julia> eigmin(A) + -1.0 + + julia> A = [0 im; -1 0] + 2×2 Array{Complex{Int64},2}: + 0+0im 0+1im + -1+0im 0+0im + + julia> eigmin(A) + ERROR: DomainError: + in #eigmin#31(::Bool, ::Bool, ::Function, ::Array{Complex{Int64},2}) at ./linalg/eigen.jl:115 + in eigmin(::Array{Complex{Int64},2}) at ./linalg/eigen.jl:113 .. function:: eigvecs(A, [eigvals,][permute=true,][scale=true]) -> Matrix @@ -636,9 +692,9 @@ Linear algebra functions in Julia are largely implemented by calling functions f .. Docstring generated from Julia source - Computes the eigenvalue decomposition of ``A``\ , returning an ``Eigen`` factorization object ``F`` which contains the eigenvalues in ``F[:values]`` and the eigenvectors in the columns of the matrix ``F[:vectors]``\ . (The ``k``\ th eigenvector can be obtained from the slice ``F[:vectors][:, k]``\ .) + Computes the eigenvalue decomposition of ``A``\ , returning an :obj:`Eigen` factorization object ``F`` which contains the eigenvalues in ``F[:values]`` and the eigenvectors in the columns of the matrix ``F[:vectors]``\ . (The ``k``\ th eigenvector can be obtained from the slice ``F[:vectors][:, k]``\ .) - The following functions are available for ``Eigen`` objects: ``inv``\ , ``det``\ . + The following functions are available for ``Eigen`` objects: :func:`inv`\ , :func:`det`\ , and :func:`isposdef`\ . If ``A`` is :class:`Symmetric`\ , :class:`Hermitian` or :class:`SymTridiagonal`\ , it is possible to calculate only a subset of the eigenvalues by specifying either a :class:`UnitRange` ``irange`` covering indices of the sorted eigenvalues or a pair ``vl`` and ``vu`` for the lower and upper boundaries of the eigenvalues. From d986e8f89a2b30d07d5368eacfb7faddfe0e2d5a Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Thu, 4 Aug 2016 06:20:24 -0700 Subject: [PATCH 57/80] excluse asc signature files from checksum listing (cherry picked from commit c3ca3dc3a34d1fc09a4962110168aeb00d8ab3e4) ref #17862 --- contrib/prepare_release.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/prepare_release.sh b/contrib/prepare_release.sh index f8d50353b7344..9f4e0f2bacb90 100755 --- a/contrib/prepare_release.sh +++ b/contrib/prepare_release.sh @@ -56,8 +56,8 @@ cp julia-$version-win32.exe julia-$majmin-latest-win32.exe echo "Note: if windows code signing is not working on the buildbots, then the" echo "checksums need to be re-calculated after the binaries are manually signed!" -shasum -a 256 julia-$version* | grep -v sha256 | grep -v md5 > julia-$version.sha256 -md5sum julia-$version* | grep -v sha256 | grep -v md5 > julia-$version.md5 +shasum -a 256 julia-$version* | grep -v -e sha256 -e md5 -e asc > julia-$version.sha256 +md5sum julia-$version* | grep -v -e sha256 -e md5 -e asc > julia-$version.md5 gpg -u julia --armor --detach-sig julia-$version-full.tar.gz gpg -u julia --armor --detach-sig julia-$version.tar.gz From f54dec2732d2ebe422cd02d649bd315f6866710d Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Thu, 4 Aug 2016 03:15:48 -0700 Subject: [PATCH 58/80] Use sphinx 1.4.5 to fix a linkcheck unicode bug with valgrind doc site (cherry picked from commit 99c8ea0f2ff857fabcc1a1852bf39302b5257af6) ref #17862 --- doc/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Makefile b/doc/Makefile index c5091f3764c04..0749d73a459a9 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -30,7 +30,7 @@ $(ACTIVATE): touch -c $@ $(SPHINX_BUILD): $(SRCDIR)/requirements.txt $(ACTIVATE) - . $(ACTIVATE) && pip install sphinx==1.3.1 \ + . $(ACTIVATE) && pip install sphinx==1.4.5 \ && pip install -r $< touch -c $@ From 1c7c90553f738ac72f912dca32bf90c4adbc5742 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Sat, 6 Aug 2016 02:31:59 -0700 Subject: [PATCH 59/80] Update url's in docs to fix `make -C doc linkcheck` (cherry picked from commit b824da4732f7e711ced3b63b1de146fc95f3121b) ref #17862 --- doc/conf.py | 3 +++ doc/devdocs/backtraces.rst | 2 +- doc/manual/calling-c-and-fortran-code.rst | 4 ++-- doc/manual/dates.rst | 2 +- doc/manual/getting-started.rst | 2 +- .../integers-and-floating-point-numbers.rst | 4 ++-- doc/manual/packages.rst | 18 +++++++++--------- doc/manual/parallel-computing.rst | 2 +- doc/manual/performance-tips.rst | 2 +- doc/stdlib/pkg.rst | 2 +- 10 files changed, 22 insertions(+), 19 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 934e10d79bb41..53dd883c7cd78 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -96,6 +96,9 @@ primary_domain = 'jl' highlight_language = 'julia' +# Flaky links to ignore in linkcheck - permissions or empty returns +linkcheck_ignore = ['https://www.appveyor.com', + 'https://bugs.kde.org/show_bug.cgi\?id=136779'] # -- Options for HTML output --------------------------------------------------- diff --git a/doc/devdocs/backtraces.rst b/doc/devdocs/backtraces.rst index 6dbe87dd3fe4e..518643397283b 100644 --- a/doc/devdocs/backtraces.rst +++ b/doc/devdocs/backtraces.rst @@ -104,4 +104,4 @@ A few terms have been used as shorthand in this guide: * ```` refers to the root directory of the Julia source tree; e.g. it should contain folders such as ``base``, ``deps``, ``src``, ``test``, etc..... .. _gist: https://gist.github.com -.. _issue: https://github.com/JuliaLang/julia/issues?state=open +.. _issue: https://github.com/JuliaLang/julia/issues?q=is%3Aopen diff --git a/doc/manual/calling-c-and-fortran-code.rst b/doc/manual/calling-c-and-fortran-code.rst index cc4fdea8a935e..535a14e8409d9 100644 --- a/doc/manual/calling-c-and-fortran-code.rst +++ b/doc/manual/calling-c-and-fortran-code.rst @@ -41,7 +41,7 @@ functions in the Julia runtime, or functions in an application linked to Julia. By default, Fortran compilers `generate mangled names -`_ +`_ (for example, converting function names to lowercase or uppercase, often appending an underscore), and so to call a Fortran function via :func:`ccall` you must pass the mangled identifier corresponding to the rule @@ -1094,7 +1094,7 @@ More About Callbacks -------------------- For more details on how to pass callbacks to C libraries, see this -`blog post `_. +`blog post `_. C++ --- diff --git a/doc/manual/dates.rst b/doc/manual/dates.rst index e23df78283255..0ede9544cf97f 100644 --- a/doc/manual/dates.rst +++ b/doc/manual/dates.rst @@ -238,7 +238,7 @@ Similarly for the :func:`monthname` function, a mapping of ``locale=>Dict{Int,St TimeType-Period Arithmetic -------------------------- -It's good practice when using any language/date framework to be familiar with how date-period arithmetic is handled as there are some `tricky issues `_ to deal with (though much less so for day-precision types). +It's good practice when using any language/date framework to be familiar with how date-period arithmetic is handled as there are some `tricky issues `_ to deal with (though much less so for day-precision types). The :mod:`Dates` module approach tries to follow the simple principle of trying to change as little as possible when doing :class:`Period` arithmetic. This approach is also often known as *calendrical* arithmetic or what you would probably guess if someone were to ask you the same calculation in a conversation. Why all the fuss about this? Let's take a classic example: add 1 month to January 31st, 2014. What's the answer? Javascript will say `March 3 `_ (assumes 31 days). PHP says `March 2 `_ (assumes 30 days). The fact is, there is no right answer. In the :mod:`Dates` module, it gives the result of February 28th. How does it figure that out? I like to think of the classic 7-7-7 gambling game in casinos. diff --git a/doc/manual/getting-started.rst b/doc/manual/getting-started.rst index d159c8731e658..d89a4bd61ad1b 100644 --- a/doc/manual/getting-started.rst +++ b/doc/manual/getting-started.rst @@ -162,7 +162,7 @@ In addition to this manual, there are various other resources that may help new users get started with Julia: - `Julia and IJulia cheatsheet `_ -- `Learn Julia in a few minutes `_ +- `Learn Julia in a few minutes `_ - `Learn Julia the Hard Way `_ - `Julia by Example `_ - `Hands-on Julia `_ diff --git a/doc/manual/integers-and-floating-point-numbers.rst b/doc/manual/integers-and-floating-point-numbers.rst index 9b37bff26eb08..0c6f3cf304f0b 100644 --- a/doc/manual/integers-and-floating-point-numbers.rst +++ b/doc/manual/integers-and-floating-point-numbers.rst @@ -554,11 +554,11 @@ computation, and also in the following references: - For even more extensive documentation of the history of, rationale for, and issues with floating-point numbers, as well as discussion of many other topics in numerical computing, see the `collected writings - `_ of `William Kahan + `_ of `William Kahan `_, commonly known as the "Father of Floating-Point". Of particular interest may be `An Interview with the Old Man of Floating-Point - `_. + `_. .. _man-arbitrary-precision-arithmetic: diff --git a/doc/manual/packages.rst b/doc/manual/packages.rst index a2db87603aa00..747fd950653dd 100644 --- a/doc/manual/packages.rst +++ b/doc/manual/packages.rst @@ -196,7 +196,7 @@ To get the latest and greatest versions of all your packages, just do :func:`Pkg The first step of updating packages is to pull new changes to ``~/.julia/v0.4/METADATA`` and see if any new registered package versions have been published. After this, :func:`Pkg.update` attempts to update packages that are checked out on a branch and not dirty (i.e. no changes have been made to files tracked by git) by pulling changes from the package's upstream repository. -Upstream changes will only be applied if no merging or rebasing is necessary – i.e. if the branch can be `"fast-forwarded" `_. +Upstream changes will only be applied if no merging or rebasing is necessary – i.e. if the branch can be `"fast-forwarded" `_. If the branch cannot be fast-forwarded, it is assumed that you're working on it and will update the repository yourself. Finally, the update process recomputes an optimal set of package versions to have installed to satisfy your top-level requirements and the requirements of "fixed" packages. @@ -360,7 +360,7 @@ We recommend that you create a `free account `_ on GitH where ``USERNAME`` is your actual GitHub user name. Once you do this, the package manager knows your GitHub user name and can configure things accordingly. -You should also `upload `_ your public SSH key to GitHub and set up an `SSH agent `_ on your development machine so that you can push changes with minimal hassle. +You should also `upload `_ your public SSH key to GitHub and set up an `SSH agent `_ on your development machine so that you can push changes with minimal hassle. In the future, we will make this system extensible and support other common git hosting options like `BitBucket `_ and allow developers to choose their favorite. Since the package development functions has been moved to the `PkgDev `_ package, you need to run ``Pkg.add("PkgDev"); import PkgDev`` to access the functions starting with ``PkgDev.`` in the document below. @@ -455,7 +455,7 @@ are several possible approaches, here is one that is widely used: ``fixbar``). By creating a branch, you ensure that you can easily go back and forth between your new work and the current ``master`` branch (see - ``_). + ``_). If you forget to do this step until after you've already made some changes, don't worry: see :ref:`more detail about branching @@ -486,7 +486,7 @@ are several possible approaches, here is one that is widely used: ``src/`` folder. -- Commit your changes: see ``_. +- Commit your changes: see ``_. - Submit your changes: From the Julia prompt, type :func:`PkgDev.submit("Foo") `. This will push your changes to your @@ -555,7 +555,7 @@ following procedure: until you have resolved the problems, or you may lose your changes. - *Reset* ``master`` (your current branch) back to an earlier state with ``git reset --hard origin/master`` (see - ``_). + ``_). This requires a bit more familiarity with git, so it's much better to get in the habit of creating a branch at the outset. @@ -583,7 +583,7 @@ quite simple but your commit history looks like this:: This gets into the territory of more advanced git usage, and you're encouraged to do some reading -(``_). However, +(``_). However, a brief summary of the procedure is as follows: - To protect yourself from error, start from your ``fixbar`` branch @@ -767,7 +767,7 @@ different license, you can ask us to add it to the package generator, or just pi If you created a GitHub account and configured git to know about it, :func:`PkgDev.generate` will set an appropriate origin URL for you. It will also automatically generate a ``.travis.yml`` file for using the `Travis `_ automated -testing service, and an ``appveyor.yml`` file for using `AppVeyor `_. You will have to enable testing on +testing service, and an ``appveyor.yml`` file for using `AppVeyor `_. You will have to enable testing on the Travis and AppVeyor websites for your package repository, but once you've done that, it will already have working tests. Of course, all the default testing does is verify that ``using FooBar`` in Julia works. @@ -840,7 +840,7 @@ on GitHub, push your changes to your fork, and open a pull request:: then you may have encountered an issue from using the GitHub API on multiple systems. The solution is to delete the "Julia Package Manager" personal access token `from your Github account - `_ and try again. + `_ and try again. Other failures may require you to circumvent :func:`PkgDev.publish` by `creating a pull request on GitHub @@ -915,7 +915,7 @@ that copy exists, you can push your local changes to your copy (just like any other GitHub project). -1. go to ``_ and create your own +1. go to ``_ and create your own fork. 2. add your fork as a remote repository for the METADATA diff --git a/doc/manual/parallel-computing.rst b/doc/manual/parallel-computing.rst index 9f9548ddf356b..d551f1e288a66 100644 --- a/doc/manual/parallel-computing.rst +++ b/doc/manual/parallel-computing.rst @@ -15,7 +15,7 @@ it's fairly obvious that a given CPU will have fastest access to the RAM within the same computer (node). Perhaps more surprisingly, similar issues are relevant on a typical multicore laptop, due to differences in the speed of main memory and the -`cache `_. Consequently, a +`cache `_. Consequently, a good multiprocessing environment should allow control over the "ownership" of a chunk of memory by a particular CPU. Julia provides a multiprocessing environment based on message passing to allow programs diff --git a/doc/manual/performance-tips.rst b/doc/manual/performance-tips.rst index e5aabd82f53c0..70d3bb9820b6b 100644 --- a/doc/manual/performance-tips.rst +++ b/doc/manual/performance-tips.rst @@ -770,7 +770,7 @@ code. Some run-time benchmarks comparing (1) type dispatch, (2) dictionary lookup, and (3) a "switch" statement can be found `on the mailing list -`_. +`_. Perhaps even worse than the run-time impact is the compile-time impact: Julia will compile specialized functions for each different diff --git a/doc/stdlib/pkg.rst b/doc/stdlib/pkg.rst index f0d455b9e0829..348505145fbb7 100644 --- a/doc/stdlib/pkg.rst +++ b/doc/stdlib/pkg.rst @@ -7,7 +7,7 @@ All package manager functions are defined in the ``Pkg`` module. None of the ``Pkg`` module's functions are exported; to use them, you'll need to prefix each function call with an explicit ``Pkg.``, e.g. ``Pkg.status()`` or ``Pkg.dir()``. -Functions for package development (e.g. ``tag``, ``publish``, etc.) have been moved to the `PkgDev `_ package. See `PkgDev README `_ for the documentation of those functions. +Functions for package development (e.g. ``tag``, ``publish``, etc.) have been moved to the `PkgDev `_ package. See `PkgDev README `_ for the documentation of those functions. .. function:: dir() -> AbstractString From 1662aaeb51fbb250cc600cb0b3f67e2c2baa0da7 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Sat, 6 Aug 2016 02:33:25 -0700 Subject: [PATCH 60/80] Update 0.4 links and Pkg output to 0.5 (backport this commit) (cherry picked from commit 25764d1fd7c41b77258842a10655e8e3213b789d) ref #17862 --- README.md | 4 ++-- doc/manual/packages.rst | 40 ++++++++++++++++++++-------------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index ff00afed1dd92..a1d390f724d6d 100644 --- a/README.md +++ b/README.md @@ -68,9 +68,9 @@ First, acquire the source code by cloning the git repository: Be sure to also configure your system to use the appropriate proxy settings, e.g. by setting the `https_proxy` and `http_proxy` variables.) -By default you will be building the latest unstable version of Julia. However, most users should use the most recent stable version of Julia, which is currently the `0.4` series of releases. You can get this version by changing to the Julia directory and running +By default you will be building the latest unstable version of Julia. However, most users should use the most recent stable version of Julia, which is currently the `0.5` series of releases. You can get this version by changing to the Julia directory and running - git checkout release-0.4 + git checkout release-0.5 Now run `make` to build the `julia` executable. To perform a parallel build, use `make -j N` and supply the maximum number of concurrent processes. (See [Platform Specific Build Notes](https://github.com/JuliaLang/julia#platform-specific-build-notes) for details.) When compiled the first time, it will automatically download and build its [external dependencies](#Required-Build-Tools-External-Libraries). diff --git a/doc/manual/packages.rst b/doc/manual/packages.rst index 747fd950653dd..e32565dafcf72 100644 --- a/doc/manual/packages.rst +++ b/doc/manual/packages.rst @@ -22,7 +22,7 @@ The :func:`Pkg.status` function prints out a summary of the state of packages yo Initially, you'll have no packages installed:: julia> Pkg.status() - INFO: Initializing package repository /Users/stefan/.julia/v0.4 + INFO: Initializing package repository /Users/stefan/.julia/v0.5 INFO: Cloning METADATA from git://github.com/JuliaLang/METADATA.jl No packages installed. @@ -56,7 +56,7 @@ This means that you tell it what you want and it figures out what versions to in So rather than installing a package, you just add it to the list of requirements and then "resolve" what needs to be installed. In particular, this means that if some package had been installed because it was needed by a previous version of something you wanted, and a newer version doesn't have that requirement anymore, updating will actually remove that package. -Your package requirements are in the file ``~/.julia/v0.4/REQUIRE``. +Your package requirements are in the file ``~/.julia/v0.5/REQUIRE``. You can edit this file by hand and then call :func:`Pkg.resolve` to install, upgrade or remove packages to optimally satisfy the requirements, or you can do :func:`Pkg.edit`, which will open ``REQUIRE`` in your editor (configured via the ``EDITOR`` or ``VISUAL`` environment variables), and then automatically call :func:`Pkg.resolve` afterwards if necessary. If you only want to add or remove the requirement for a single package, you can also use the non-interactive :func:`Pkg.add` and :func:`Pkg.rm` commands, which add or remove a single requirement to ``REQUIRE`` and then call :func:`Pkg.resolve`. @@ -81,15 +81,15 @@ You can add a package to the list of requirements with the :func:`Pkg.add` funct - NumericExtensions 0.2.17 - Stats 0.2.6 -What this is doing is first adding ``Distributions`` to your ``~/.julia/v0.4/REQUIRE`` file:: +What this is doing is first adding ``Distributions`` to your ``~/.julia/v0.5/REQUIRE`` file:: - $ cat ~/.julia/v0.4/REQUIRE + $ cat ~/.julia/v0.5/REQUIRE Distributions It then runs :func:`Pkg.resolve` using these new requirements, which leads to the conclusion that the ``Distributions`` package should be installed since it is required but not installed. -As stated before, you can accomplish the same thing by editing your ``~/.julia/v0.4/REQUIRE`` file by hand and then running :func:`Pkg.resolve` yourself:: +As stated before, you can accomplish the same thing by editing your ``~/.julia/v0.5/REQUIRE`` file by hand and then running :func:`Pkg.resolve` yourself:: - $ echo UTF16 >> ~/.julia/v0.4/REQUIRE + $ echo UTF16 >> ~/.julia/v0.5/REQUIRE julia> Pkg.resolve() INFO: Cloning cache of UTF16 from git://github.com/nolta/UTF16.jl.git @@ -175,7 +175,7 @@ To install an unregistered package, use :func:`Pkg.clone(url) `, wher Resolving deltas: 100% (8/8), done. By convention, Julia repository names end with ``.jl`` (the additional ``.git`` indicates a "bare" git repository), which keeps them from colliding with repositories for other languages, and also makes Julia packages easy to find in search engines. -When packages are installed in your ``.julia/v0.4`` directory, however, the extension is redundant so we leave it off. +When packages are installed in your ``.julia/v0.5`` directory, however, the extension is redundant so we leave it off. If unregistered packages contain a ``REQUIRE`` file at the top of their source tree, that file will be used to determine which registered packages the unregistered package depends on, and they will automatically be installed. Unregistered packages participate in the same version resolution logic as registered packages, so installed package versions will be adjusted as necessary to satisfy the requirements of both registered and unregistered packages. @@ -194,7 +194,7 @@ To get the latest and greatest versions of all your packages, just do :func:`Pkg INFO: Upgrading Distributions: v0.2.8 => v0.2.10 INFO: Upgrading Stats: v0.2.7 => v0.2.8 -The first step of updating packages is to pull new changes to ``~/.julia/v0.4/METADATA`` and see if any new registered package versions have been published. +The first step of updating packages is to pull new changes to ``~/.julia/v0.5/METADATA`` and see if any new registered package versions have been published. After this, :func:`Pkg.update` attempts to update packages that are checked out on a branch and not dirty (i.e. no changes have been made to files tracked by git) by pulling changes from the package's upstream repository. Upstream changes will only be applied if no merging or rebasing is necessary – i.e. if the branch can be `"fast-forwarded" `_. If the branch cannot be fast-forwarded, it is assumed that you're working on it and will update the repository yourself. @@ -207,7 +207,7 @@ A package is considered fixed if it is one of the following: 3. **Dirty:** changes have been made to files in the repo. If any of these are the case, the package manager cannot freely change the installed version of the package, so its requirements must be satisfied by whatever other package versions it picks. -The combination of top-level requirements in ``~/.julia/v0.4/REQUIRE`` and the requirement of fixed packages are used to determine what should be installed. +The combination of top-level requirements in ``~/.julia/v0.5/REQUIRE`` and the requirement of fixed packages are used to determine what should be installed. You can also update only a subset of the installed packages, by providing arguments to the `Pkg.update` function. In that case, only the packages provided as arguments and their dependencies will be updated:: @@ -720,7 +720,7 @@ Suppose you want to create a new Julia package called ``FooBar``. To get starte name of a license that the package generator knows about:: julia> PkgDev.generate("FooBar","MIT") - INFO: Initializing FooBar repo: /Users/stefan/.julia/v0.4/FooBar + INFO: Initializing FooBar repo: /Users/stefan/.julia/v0.5/FooBar INFO: Origin: git://github.com/StefanKarpinski/FooBar.jl.git INFO: Generating LICENSE.md INFO: Generating README.md @@ -732,10 +732,10 @@ name of a license that the package generator knows about:: INFO: Generating .gitignore INFO: Committing FooBar generated files -This creates the directory ``~/.julia/v0.4/FooBar``, initializes it as a git repository, generates a bunch of files +This creates the directory ``~/.julia/v0.5/FooBar``, initializes it as a git repository, generates a bunch of files that all packages should have, and commits them to the repository:: - $ cd ~/.julia/v0.4/FooBar && git show --stat + $ cd ~/.julia/v0.5/FooBar && git show --stat commit 84b8e266dae6de30ab9703150b3bf771ec7b6285 Author: Stefan Karpinski @@ -763,7 +763,7 @@ that all packages should have, and commits them to the repository:: At the moment, the package manager knows about the MIT "Expat" License, indicated by ``"MIT"``, the Simplified BSD License, indicated by ``"BSD"``, and version 2.0 of the Apache Software License, indicated by ``"ASL"``. If you want to use a different license, you can ask us to add it to the package generator, or just pick one of these three and then modify the -``~/.julia/v0.4/PACKAGE/LICENSE.md`` file after it has been generated. +``~/.julia/v0.5/PACKAGE/LICENSE.md`` file after it has been generated. If you created a GitHub account and configured git to know about it, :func:`PkgDev.generate` will set an appropriate origin URL for you. It will also automatically generate a ``.travis.yml`` file for using the `Travis `_ automated @@ -799,9 +799,9 @@ of ``METADATA`` using :func:`PkgDev.register`:: INFO: Registering FooBar at git://github.com/StefanKarpinski/FooBar.jl.git INFO: Committing METADATA for FooBar -This creates a commit in the ``~/.julia/v0.4/METADATA`` repo:: +This creates a commit in the ``~/.julia/v0.5/METADATA`` repo:: - $ cd ~/.julia/v0.4/METADATA && git show + $ cd ~/.julia/v0.5/METADATA && git show commit 9f71f4becb05cadacb983c54a72eed744e5c019d Author: Stefan Karpinski @@ -857,12 +857,12 @@ register it with the :func:`PkgDev.tag` command:: This tags ``v0.0.1`` in the ``FooBar`` repo:: - $ cd ~/.julia/v0.4/FooBar && git tag + $ cd ~/.julia/v0.5/FooBar && git tag v0.0.1 It also creates a new version entry in your local ``METADATA`` repo for ``FooBar``:: - $ cd ~/.julia/v0.4/FooBar && git show + $ cd ~/.julia/v0.5/FooBar && git show commit de77ee4dc0689b12c5e8b574aef7f70e8b311b0e Author: Stefan Karpinski Date: Wed Oct 16 23:06:18 2013 -0400 @@ -922,7 +922,7 @@ fork. repository on your local computer (in the terminal where USERNAME is your github username):: - cd ~/.julia/v0.4/METADATA + cd ~/.julia/v0.5/METADATA git remote add USERNAME https://github.com/USERNAME/METADATA.jl.git 3. push your changes to your fork:: @@ -938,7 +938,7 @@ Fixing Package Requirements If you need to fix the registered requirements of an already-published package version, you can do so just by editing the metadata for that version, which will still have the same commit hash – the hash associated with a version is permanent:: - $ cd ~/.julia/v0.4/METADATA/FooBar/versions/0.0.1 && cat requires + $ cd ~/.julia/v0.5/METADATA/FooBar/versions/0.0.1 && cat requires julia 0.3- $ vi requires @@ -951,7 +951,7 @@ When you fix the requirements in ``METADATA`` for a previous version of a packag Requirements Specification -------------------------- -The ``~/.julia/v0.4/REQUIRE`` file, the ``REQUIRE`` file inside packages, and the ``METADATA`` package ``requires`` files use a simple line-based format to express the ranges of package versions which need to be installed. Package ``REQUIRE`` and ``METADATA requires`` files should also include the range of versions of ``julia`` the package is expected to work with. +The ``~/.julia/v0.5/REQUIRE`` file, the ``REQUIRE`` file inside packages, and the ``METADATA`` package ``requires`` files use a simple line-based format to express the ranges of package versions which need to be installed. Package ``REQUIRE`` and ``METADATA requires`` files should also include the range of versions of ``julia`` the package is expected to work with. Here's how these files are parsed and interpreted. From a4ee378376872b34108e7ce8f83f32897f1d5ed9 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Tue, 9 Aug 2016 12:30:08 -0700 Subject: [PATCH 61/80] doc formatting and add back threadcall note (cherry picked from commit fa7f995c0a17e527e1d4f44f7c1b96077e26224e) ref #17862 --- base/threadcall.jl | 2 ++ doc/manual/parallel-computing.rst | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/base/threadcall.jl b/base/threadcall.jl index c9da92c5a49ea..be3e99f324cb8 100644 --- a/base/threadcall.jl +++ b/base/threadcall.jl @@ -23,6 +23,8 @@ function without causing the main `julia` thread to become blocked. Concurrency is limited by size of the libuv thread pool, which defaults to 4 threads but can be increased by setting the `UV_THREADPOOL_SIZE` environment variable and restarting the `julia` process. + +Note that the called function should never call back into Julia. """ macro threadcall(f, rettype, argtypes, argvals...) # check for usage errors diff --git a/doc/manual/parallel-computing.rst b/doc/manual/parallel-computing.rst index d551f1e288a66..588a364ff997a 100644 --- a/doc/manual/parallel-computing.rst +++ b/doc/manual/parallel-computing.rst @@ -1081,21 +1081,21 @@ Note that :obj:`Threads.@threads` does not have an optional reduction parameter All I/O tasks, timers, REPL commands, etc are multiplexed onto a single OS thread via an event loop. A patched version of libuv (http://docs.libuv.org/en/v1.x/) provides this functionality. Yield points provide for co-operatively scheduling multiple tasks onto the same OS thread. I/O tasks and timers yield implicitly while -waiting for the event to occur. Calling `yield()` explicitly allows for other tasks to be scheduled. +waiting for the event to occur. Calling ``yield()`` explicitly allows for other tasks to be scheduled. Thus, a task executing a ``ccall`` effectively prevents the Julia scheduler from executing any other tasks till the call returns. This is true for all calls into external libraries. Exceptions are calls into -custom C code that call back into Julia (which may then yield) or C code that calls ``jl_yield()``(C equivalent of ``yield()``). +custom C code that call back into Julia (which may then yield) or C code that calls ``jl_yield()`` (C equivalent of ``yield()``). Note that while Julia code runs on a single thread (by default), libraries used by Julia may launch their own internal threads. For example, the BLAS library may start as many threads as there are cores on a machine. The ``@threadcall`` macro addresses scenarios where we do not want a ``ccall`` to block the main Julia event loop. It schedules a C function for execution in a separate thread. A threadpool with a default size of 4 is used for this. -The size of the threadpool is controlled via environment variable UV_THREADPOOL_SIZE. While waiting for a free thread, +The size of the threadpool is controlled via environment variable ``UV_THREADPOOL_SIZE``. While waiting for a free thread, and during function execution once a thread is available, the requesting task (on the main Julia event loop) yields to other tasks. Note that ``@threadcall`` does not return till the execution is complete. From a user point of -view, it is therefore a blocking call like other Julia API. +view, it is therefore a blocking call like other Julia APIs. It is very important that the called function does not call back into Julia. From 203a8f01274636e8cf2c8538c0a1447990b0d61c Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 9 Aug 2016 16:06:01 -0400 Subject: [PATCH 62/80] fix spawn test to check exit status of async tasks (cherry picked from commit 8ea8df46e9a192d02fb51b6f2a6a9282b99a56e4) ref #17925 --- test/spawn.jl | 173 +++++++++++++++++++++++++++----------------------- 1 file changed, 95 insertions(+), 78 deletions(-) diff --git a/test/spawn.jl b/test/spawn.jl index cfff6b6b6425a..21ce279b5eb3e 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -83,27 +83,32 @@ end @test_broken success(ignorestatus(falsecmd & falsecmd)) # STDIN Redirection -file = tempname() -run(pipeline(`$echo hello world`, file)) -@test readstring(pipeline(file, catcmd)) == "hello world\n" -@test open(readstring, pipeline(file, catcmd), "r") == "hello world\n" -rm(file) +let file = tempname() + run(pipeline(`$echo hello world`, file)) + @test readstring(pipeline(file, catcmd)) == "hello world\n" + @test open(readstring, pipeline(file, catcmd), "r") == "hello world\n" + rm(file) +end # Stream Redirection if !is_windows() # WINNT reports operation not supported on socket (ENOTSUP) for this test - local r = Channel(1), port, server, sock, client - @async begin + local r = Channel(1), port, server, sock, client, t1, t2 + t1 = @async begin port, server = listenany(2326) put!(r, port) client = accept(server) @test readstring(pipeline(client, catcmd)) == "hello world\n" close(server) + return true end - @async begin + t2 = @async begin sock = connect(fetch(r)) run(pipeline(`$echo hello world`, sock)) close(sock) + return true end + @test wait(t1) + @test wait(t2) end @test readstring(setenv(`$shcmd -c "echo \$TEST"`,["TEST=Hello World"])) == "Hello World\n" @@ -111,77 +116,87 @@ end @test readstring(setenv(`$shcmd -c "echo \$TEST"`,"TEST"=>"Hello World")) == "Hello World\n" @test (withenv("TEST"=>"Hello World") do readstring(`$shcmd -c "echo \$TEST"`); end) == "Hello World\n" -pathA = readchomp(setenv(`$shcmd -c "pwd -P"`;dir="..")) -pathB = readchomp(setenv(`$shcmd -c "cd .. && pwd -P"`)) -if is_windows() - # on windows, sh returns posix-style paths that are not valid according to ispath - @test pathA == pathB -else - @test Base.samefile(pathA, pathB) +let pathA = readchomp(setenv(`$shcmd -c "pwd -P"`;dir="..")), + pathB = readchomp(setenv(`$shcmd -c "cd .. && pwd -P"`)) + if is_windows() + # on windows, sh returns posix-style paths that are not valid according to ispath + @test pathA == pathB + else + @test Base.samefile(pathA, pathB) + end end -# Here we test that if we close a stream with pending writes, we don't lose the writes. -str = "" -for i=1:1000 - str = "$str\n $(randstring(10))" -end -stdout, stdin, proc = readandwrite(`$catcmd -`) -write(stdin, str) -close(stdin) -str2 = readstring(stdout) -@test str2 == str - -# This test hangs if the end of run walk across uv streams calls shutdown on a stream that is shutting down. -file = tempname() -open(pipeline(`$catcmd -`, file), "w") do io - write(io, str) +let str = "", stdin, stdout, proc, str2, file + for i = 1:1000 + str = "$str\n $(randstring(10))" + end + + # Here we test that if we close a stream with pending writes, we don't lose the writes. + stdout, stdin, proc = readandwrite(`$catcmd -`) + write(stdin, str) + close(stdin) + str2 = readstring(stdout) + @test str2 == str + + # This test hangs if the end-of-run-walk-across-uv-streams calls shutdown on a stream that is shutting down. + file = tempname() + open(pipeline(`$catcmd -`, file), "w") do io + write(io, str) + end + rm(file) end -rm(file) # issue #3373 # fixing up Conditions after interruptions -r = Channel(1) -t = @async begin - try - wait(r) +let r, t + r = Channel(1) + t = @async begin + try + wait(r) + end + p = spawn(`$sleepcmd 1`); wait(p) + @test p.exitcode == 0 + return true end - p = spawn(`$sleepcmd 1`); wait(p) - @test p.exitcode == 0 + yield() + schedule(t, InterruptException(), error=true) + yield() + put!(r,11) + yield() + @test wait(t) end -yield() -schedule(t, InterruptException(), error=true) -yield() -put!(r,11) -yield() # Test marking of IO -r = Channel(1) -@async begin - port, server = listenany(2327) - put!(r, port) - client = accept(server) - write(client, "Hello, world!\n") - write(client, "Goodbye, world...\n") - close(server) +let r, t, sock + r = Channel(1) + t = @async begin + port, server = listenany(2327) + put!(r, port) + client = accept(server) + write(client, "Hello, world!\n") + write(client, "Goodbye, world...\n") + close(server) + return true + end + sock = connect(fetch(r)) + mark(sock) + @test ismarked(sock) + @test readline(sock) == "Hello, world!\n" + @test readline(sock) == "Goodbye, world...\n" + @test reset(sock) == 0 + @test !ismarked(sock) + mark(sock) + @test ismarked(sock) + @test readline(sock) == "Hello, world!\n" + unmark(sock) + @test !ismarked(sock) + @test_throws ArgumentError reset(sock) + @test !unmark(sock) + @test readline(sock) == "Goodbye, world...\n" + #@test eof(sock) ## doesn't work + close(sock) + @test wait(t) end -sock = connect(fetch(r)) -mark(sock) -@test ismarked(sock) -@test readline(sock) == "Hello, world!\n" -@test readline(sock) == "Goodbye, world...\n" -@test reset(sock) == 0 -@test !ismarked(sock) -mark(sock) -@test ismarked(sock) -@test readline(sock) == "Hello, world!\n" -unmark(sock) -@test !ismarked(sock) -@test_throws ArgumentError reset(sock) -@test !unmark(sock) -@test readline(sock) == "Goodbye, world...\n" -#@test eof(sock) ## doesn't work -close(sock) - # issue #4535 exename = Base.julia_cmd() if valgrind_off @@ -210,16 +225,18 @@ yield() @test consume(ducer) == 2 # redirect_* -OLD_STDOUT = STDOUT -fname = tempname() -f = open(fname,"w") -redirect_stdout(f) -println("Hello World") -redirect_stdout(OLD_STDOUT) -close(f) -@test "Hello World\n" == readstring(fname) -@test is(OLD_STDOUT,STDOUT) -rm(fname) +let OLD_STDOUT = STDOUT, + fname = tempname(), + f = open(fname,"w") + + redirect_stdout(f) + println("Hello World") + redirect_stdout(OLD_STDOUT) + close(f) + @test "Hello World\n" == readstring(fname) + @test is(OLD_STDOUT,STDOUT) + rm(fname) +end # Test that redirecting an IOStream does not crash the process let fname = tempname() From 032db5fdcbac08c3d5b7eac792bbb3248d63b3dc Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Wed, 27 Jul 2016 16:25:50 -0400 Subject: [PATCH 63/80] fix #13529, slowdown with large number of async `sleep` calls The problem was performance degradation of ObjectIdDict with many deleted items. The table needs to be rehashed after a large number of deletions. (cherry picked from commit c5e0f47a57993aae6c4ed3fa310724064fc51d16) ref #17655 --- base/dict.jl | 24 +++++++++++++++++++----- src/dump.c | 2 +- src/julia_internal.h | 2 +- src/table.c | 14 ++++++++------ 4 files changed, 29 insertions(+), 13 deletions(-) diff --git a/base/dict.jl b/base/dict.jl index 2ea7e983b1c85..d66b93377c7f8 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -247,7 +247,8 @@ push!(t::Associative, p::Pair, q::Pair, r::Pair...) = push!(push!(push!(t, p), q type ObjectIdDict <: Associative{Any,Any} ht::Vector{Any} - ObjectIdDict() = new(Vector{Any}(32)) + ndel::Int + ObjectIdDict() = new(Vector{Any}(32), 0) function ObjectIdDict(itr) d = ObjectIdDict() @@ -266,7 +267,16 @@ end similar(d::ObjectIdDict) = ObjectIdDict() +function rehash!(t::ObjectIdDict, newsz = length(t.ht)) + t.ht = ccall(:jl_idtable_rehash, Any, (Any, Csize_t), t.ht, newsz) + t +end + function setindex!(t::ObjectIdDict, v::ANY, k::ANY) + if t.ndel >= ((3*length(t.ht))>>2) + rehash!(t, max(length(t.ht)>>1, 32)) + t.ndel = 0 + end t.ht = ccall(:jl_eqtable_put, Array{Any,1}, (Any, Any, Any), t.ht, k, v) return t end @@ -274,8 +284,12 @@ end get(t::ObjectIdDict, key::ANY, default::ANY) = ccall(:jl_eqtable_get, Any, (Any, Any, Any), t.ht, key, default) -pop!(t::ObjectIdDict, key::ANY, default::ANY) = - ccall(:jl_eqtable_pop, Any, (Any, Any, Any), t.ht, key, default) +function pop!(t::ObjectIdDict, key::ANY, default::ANY) + val = ccall(:jl_eqtable_pop, Any, (Any, Any, Any), t.ht, key, default) + # TODO: this can underestimate `ndel` + val === default || (t.ndel += 1) + return val +end function pop!(t::ObjectIdDict, key::ANY) val = pop!(t, key, secret_table_token) @@ -283,11 +297,11 @@ function pop!(t::ObjectIdDict, key::ANY) end function delete!(t::ObjectIdDict, key::ANY) - ccall(:jl_eqtable_pop, Any, (Any, Any), t.ht, key) + pop!(t, key, secret_table_token) t end -empty!(t::ObjectIdDict) = (t.ht = Vector{Any}(length(t.ht)); t) +empty!(t::ObjectIdDict) = (t.ht = Vector{Any}(length(t.ht)); t.ndel = 0; t) _oidd_nextind(a, i) = reinterpret(Int,ccall(:jl_eqtable_nextind, Csize_t, (Any, Csize_t), a, i)) diff --git a/src/dump.c b/src/dump.c index 7157d89bb5a36..d46fb1b624d7a 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1815,7 +1815,7 @@ static void jl_reinit_item(jl_value_t *v, int how, arraylist_t *tracee_list) case 1: { // rehash ObjectIdDict jl_array_t **a = (jl_array_t**)v; // Assume *a don't need a write barrier - jl_idtable_rehash(a, jl_array_len(*a)); + *a = jl_idtable_rehash(*a, jl_array_len(*a)); jl_gc_wb(v, *a); break; } diff --git a/src/julia_internal.h b/src/julia_internal.h index 2c7105b62e53c..8346c247e5808 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -425,7 +425,7 @@ int32_t jl_get_llvm_gv(jl_value_t *p); int32_t jl_assign_functionID(/*llvm::Function*/void *function); // the first argument to jl_idtable_rehash is used to return a value // make sure it is rooted if it is used after the function returns -void jl_idtable_rehash(jl_array_t **pa, size_t newsz); +JL_DLLEXPORT jl_array_t *jl_idtable_rehash(jl_array_t *a, size_t newsz); JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *module); jl_lambda_info_t *jl_get_specialization1(jl_tupletype_t *types); diff --git a/src/table.c b/src/table.c index 5bed49efbfe26..37738f9816d71 100644 --- a/src/table.c +++ b/src/table.c @@ -10,13 +10,13 @@ static void **jl_table_lookup_bp(jl_array_t **pa, void *key); -void jl_idtable_rehash(jl_array_t **pa, size_t newsz) +JL_DLLEXPORT jl_array_t *jl_idtable_rehash(jl_array_t *a, size_t newsz) { // Assume *pa don't need a write barrier // pa doesn't have to be a GC slot but *pa needs to be rooted - size_t sz = jl_array_len(*pa); + size_t sz = jl_array_len(a); size_t i; - void **ol = (void**)(*pa)->data; + void **ol = (void**)a->data; jl_array_t *newa = jl_alloc_vec_any(newsz); // keep the original array in the original slot since we need `ol` // to be valid in the loop below. @@ -25,16 +25,16 @@ void jl_idtable_rehash(jl_array_t **pa, size_t newsz) if (ol[i+1] != NULL) { (*jl_table_lookup_bp(&newa, ol[i])) = ol[i+1]; jl_gc_wb(newa, ol[i+1]); - // it is however necessary here because allocation + // it is however necessary here because allocation // can (and will) occur in a recursive call inside table_lookup_bp } } - *pa = newa; // we do not check the write barrier here // because pa always points to a C stack location // (see jl_eqtable_put and jl_finalize_deserializer) // it should be changed if this assumption no longer holds JL_GC_POP(); + return newa; } static void **jl_table_lookup_bp(jl_array_t **pa, void *key) @@ -44,6 +44,7 @@ static void **jl_table_lookup_bp(jl_array_t **pa, void *key) jl_array_t *a = *pa; size_t orig, index, iter; size_t newsz, sz = hash_size(a); + assert(sz >= 1); size_t maxprobe = max_probe(sz); void **tab = (void**)a->data; @@ -81,7 +82,7 @@ static void **jl_table_lookup_bp(jl_array_t **pa, void *key) newsz = HT_N_INLINE; else newsz = sz<<2; - jl_idtable_rehash(pa, newsz); + *pa = jl_idtable_rehash(*pa, newsz); a = *pa; tab = (void**)a->data; @@ -98,6 +99,7 @@ static void **jl_table_lookup_bp(jl_array_t **pa, void *key) static void **jl_table_peek_bp(jl_array_t *a, void *key) { size_t sz = hash_size(a); + assert(sz >= 1); size_t maxprobe = max_probe(sz); void **tab = (void**)a->data; uint_t hv = keyhash((jl_value_t*)key); From b3fd7955a87ac9c93bcb1e35675e40659a6d46b1 Mon Sep 17 00:00:00 2001 From: Art Date: Wed, 10 Aug 2016 02:20:02 -0400 Subject: [PATCH 64/80] fix incorrect bignum rng buffer size [fix #17772] (#17874) (cherry picked from commit 99c4add7e8848ead56e59e63c462ac795024d66a) --- deps/mbedtls.mk | 6 +-- deps/patches/libssh2-mbedtls.patch | 85 ++++++++++++++++++++++-------- deps/patches/mbedtls-config.patch | 27 ---------- 3 files changed, 63 insertions(+), 55 deletions(-) delete mode 100644 deps/patches/mbedtls-config.patch diff --git a/deps/mbedtls.mk b/deps/mbedtls.mk index 79c8513a44579..1ed14a48ab8a0 100644 --- a/deps/mbedtls.mk +++ b/deps/mbedtls.mk @@ -35,11 +35,7 @@ $(SRCDIR)/srccache/$(MBEDTLS_SRC)/CMakeLists.txt: $(SRCDIR)/srccache/$(MBEDTLS_S $(TAR) -C $(dir $@) --strip-components 1 -xf $< touch -c $@ -$(SRCDIR)/srccache/$(MBEDTLS_SRC)/mbedtls-config.patch-applied: | $(SRCDIR)/srccache/$(MBEDTLS_SRC)/CMakeLists.txt - cd $(SRCDIR)/srccache/$(MBEDTLS_SRC) && patch -p0 -f < $(SRCDIR)/patches/mbedtls-config.patch - echo 1 > $@ - -$(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/Makefile: $(SRCDIR)/srccache/$(MBEDTLS_SRC)/CMakeLists.txt $(SRCDIR)/srccache/$(MBEDTLS_SRC)/mbedtls-config.patch-applied +$(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/Makefile: $(SRCDIR)/srccache/$(MBEDTLS_SRC)/CMakeLists.txt mkdir -p $(dir $@) cd $(dir $@) && \ $(CMAKE) $(dir $<) $(MBEDTLS_OPTS) diff --git a/deps/patches/libssh2-mbedtls.patch b/deps/patches/libssh2-mbedtls.patch index 2311d1e210b49..d7cdeef91a44b 100644 --- a/deps/patches/libssh2-mbedtls.patch +++ b/deps/patches/libssh2-mbedtls.patch @@ -159,10 +159,10 @@ index e85aecd..366d007 100644 diff --git a/src/mbedtls.c b/src/mbedtls.c new file mode 100644 -index 0000000..98bc549 +index 0000000..1d181e1 --- /dev/null +++ b/src/mbedtls.c -@@ -0,0 +1,570 @@ +@@ -0,0 +1,606 @@ +#include "libssh2_priv.h" + +#ifdef LIBSSH2_MBEDTLS /* compile only if we build with mbedtls */ @@ -277,10 +277,10 @@ index 0000000..98bc549 + if(!ret) + ret = mbedtls_cipher_finish(ctx, output + olen, &finish_olen); + -+ olen += finish_olen; -+ -+ if (!ret) ++ if (!ret) { ++ olen += finish_olen; + memcpy(block, output, olen); ++ } + + _libssh2_mbedtls_safe_free(output, osize); + } @@ -306,6 +306,9 @@ index 0000000..98bc549 + int ret, hmac; + + md_info = mbedtls_md_info_from_type(mdtype); ++ if(!md_info) ++ return 0; ++ + hmac = key == NULL ? 0 : 1; + + mbedtls_md_init(ctx); @@ -339,6 +342,9 @@ index 0000000..98bc549 + int ret; + + md_info = mbedtls_md_info_from_type(mdtype); ++ if(!md_info) ++ return 0; ++ + ret = mbedtls_md(md_info, data, datalen, hash); + + return ret == 0 ? 0 : -1; @@ -362,17 +368,47 @@ index 0000000..98bc549 + return bignum; +} + -+void -+_libssh2_mbedtls_bignum_free(_libssh2_bn *bn) ++int ++_libssh2_mbedtls_bignum_random(_libssh2_bn *bn, int bits, int top, int bottom) +{ -+ if (bn) -+ { -+ mbedtls_mpi_free(bn); -+#ifdef LIBSSH2_CLEAR_MEMORY -+ memset(bn, 0, sizeof(_libssh2_bn)); -+#endif ++ size_t len; ++ int err; ++ int i; ++ ++ if (!bn || bits <= 0) ++ return -1; ++ ++ len = (bits + 7) >> 3; ++ err = mbedtls_mpi_fill_random(bn, len, mbedtls_ctr_drbg_random, &_libssh2_mbedtls_ctr_drbg); ++ if (err) ++ return -1; ++ ++ /* Zero unsued bits above the most significant bit*/ ++ for(i=len*8-1;bits<=i;--i) { ++ err = mbedtls_mpi_set_bit(bn, i, 0); ++ if (err) ++ return -1; ++ } ++ ++ /* If `top` is -1, the most significant bit of the random number can be zero. ++ If top is 0, the most significant bit of the random number is set to 1, ++ and if top is 1, the two most significant bits of the number will be set ++ to 1, so that the product of two such random numbers will always have 2*bits length. ++ */ ++ for(i=0;i<=top;++i) { ++ err = mbedtls_mpi_set_bit(bn, bits-i-1, 1); ++ if (err) ++ return -1; ++ } ++ ++ /* make odd by setting first bit in least significant byte */ ++ if (bottom) { ++ err = mbedtls_mpi_set_bit(bn, 0, 1); ++ if (err) ++ return -1; + } -+ mbedtls_free(bn); ++ ++ return 0; +} + + @@ -464,7 +500,7 @@ index 0000000..98bc549 + mbedtls_pk_init(&pkey); + + ret = mbedtls_pk_parse_keyfile(&pkey, filename, (char *)passphrase); -+ if( ret != 0 ) ++ if( ret != 0 || mbedtls_pk_get_type(&pkey) != MBEDTLS_PK_RSA) + { + mbedtls_pk_free(&pkey); + mbedtls_rsa_free(*rsa); @@ -498,7 +534,7 @@ index 0000000..98bc549 + + ret = mbedtls_pk_parse_key(&pkey, (unsigned char *)filedata, + filedata_len, NULL, 0); -+ if( ret != 0 ) ++ if( ret != 0 || mbedtls_pk_get_type(&pkey) != MBEDTLS_PK_RSA) + { + mbedtls_pk_free(&pkey); + mbedtls_rsa_free(*rsa); @@ -529,7 +565,7 @@ index 0000000..98bc549 + return -1; /* failure */ + + ret = mbedtls_rsa_pkcs1_verify(rsa, NULL, NULL, MBEDTLS_RSA_PUBLIC, -+ MBEDTLS_MD_SHA1, sig_len, hash, sig); ++ MBEDTLS_MD_SHA1, SHA_DIGEST_LENGTH, hash, sig); + + return (ret == 0) ? 0 : -1; +} @@ -735,10 +771,10 @@ index 0000000..98bc549 +#endif /* LIBSSH2_MBEDTLS */ diff --git a/src/mbedtls.h b/src/mbedtls.h new file mode 100644 -index 0000000..f594575 +index 0000000..248583e --- /dev/null +++ b/src/mbedtls.h -@@ -0,0 +1,368 @@ +@@ -0,0 +1,371 @@ +#include +#include + @@ -980,8 +1016,8 @@ index 0000000..f594575 + _libssh2_mbedtls_bignum_init() +#define _libssh2_bn_init_from_bin() \ + _libssh2_mbedtls_bignum_init() -+#define _libssh2_bn_rand(bn, bytes, top, bottom) \ -+ mbedtls_mpi_fill_random(bn, bytes, mbedtls_ctr_drbg_random, &_libssh2_mbedtls_ctr_drbg) ++#define _libssh2_bn_rand(bn, bits, top, bottom) \ ++ _libssh2_mbedtls_bignum_random(bn, bits, top, bottom) +#define _libssh2_bn_mod_exp(r, a, p, m, ctx) \ + mbedtls_mpi_exp_mod(r, a, p, m, NULL) +#define _libssh2_bn_set_word(bn, word) \ @@ -995,7 +1031,7 @@ index 0000000..f594575 +#define _libssh2_bn_bits(bn) \ + mbedtls_mpi_bitlen(bn) +#define _libssh2_bn_free(bn) \ -+ _libssh2_mbedtls_bignum_free(bn) ++ mbedtls_mpi_free(bn) + + +/*******************************************************************/ @@ -1044,6 +1080,9 @@ index 0000000..f594575 +_libssh2_mbedtls_bignum_free(_libssh2_bn *bn); + +int ++_libssh2_mbedtls_bignum_random(_libssh2_bn *bn, int bits, int top, int bottom); ++ ++int +_libssh2_mbedtls_rsa_new(libssh2_rsa_ctx **rsa, + const unsigned char *edata, + unsigned long elen, @@ -1106,4 +1145,4 @@ index 0000000..f594575 + size_t *pubkeydata_len, + const char *privatekeydata, + size_t privatekeydata_len, -+ const char *passphrase); ++ const char *passphrase); \ No newline at end of file diff --git a/deps/patches/mbedtls-config.patch b/deps/patches/mbedtls-config.patch deleted file mode 100644 index 1f774ec61ac33..0000000000000 --- a/deps/patches/mbedtls-config.patch +++ /dev/null @@ -1,27 +0,0 @@ -diff --git include/mbedtls/config.h include/mbedtls/config.h -index 0efee04..787cd8c 100644 ---- include/mbedtls/config.h -+++ include/mbedtls/config.h -@@ -2434,19 +2434,19 @@ - - /* MPI / BIGNUM options */ - //#define MBEDTLS_MPI_WINDOW_SIZE 6 /**< Maximum windows size used. */ --//#define MBEDTLS_MPI_MAX_SIZE 1024 /**< Maximum number of bytes for usable MPIs. */ -+#define MBEDTLS_MPI_MAX_SIZE 2048 /**< Maximum number of bytes for usable MPIs. */ - - /* CTR_DRBG options */ - //#define MBEDTLS_CTR_DRBG_ENTROPY_LEN 48 /**< Amount of entropy used per seed by default (48 with SHA-512, 32 with SHA-256) */ - //#define MBEDTLS_CTR_DRBG_RESEED_INTERVAL 10000 /**< Interval before reseed is performed by default */ - //#define MBEDTLS_CTR_DRBG_MAX_INPUT 256 /**< Maximum number of additional input bytes */ --//#define MBEDTLS_CTR_DRBG_MAX_REQUEST 1024 /**< Maximum number of requested bytes per call */ -+#define MBEDTLS_CTR_DRBG_MAX_REQUEST 2048 /**< Maximum number of requested bytes per call */ - //#define MBEDTLS_CTR_DRBG_MAX_SEED_INPUT 384 /**< Maximum size of (re)seed buffer */ - - /* HMAC_DRBG options */ - //#define MBEDTLS_HMAC_DRBG_RESEED_INTERVAL 10000 /**< Interval before reseed is performed by default */ - //#define MBEDTLS_HMAC_DRBG_MAX_INPUT 256 /**< Maximum number of additional input bytes */ --//#define MBEDTLS_HMAC_DRBG_MAX_REQUEST 1024 /**< Maximum number of requested bytes per call */ -+#define MBEDTLS_HMAC_DRBG_MAX_REQUEST 2048 /**< Maximum number of requested bytes per call */ - //#define MBEDTLS_HMAC_DRBG_MAX_SEED_INPUT 384 /**< Maximum size of (re)seed buffer */ - - /* ECP options */ From b39e59dccdd9e80f87d84f2a491c3ec77c745cf1 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Wed, 10 Aug 2016 02:48:52 -0400 Subject: [PATCH 65/80] Patch is picky about newlines at end of file (cherry picked from commit cbb1b9647ee84e46b895db08ca98ad1b77d890f1) --- deps/patches/libssh2-mbedtls.patch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/patches/libssh2-mbedtls.patch b/deps/patches/libssh2-mbedtls.patch index d7cdeef91a44b..63856f33a9e79 100644 --- a/deps/patches/libssh2-mbedtls.patch +++ b/deps/patches/libssh2-mbedtls.patch @@ -1145,4 +1145,4 @@ index 0000000..248583e + size_t *pubkeydata_len, + const char *privatekeydata, + size_t privatekeydata_len, -+ const char *passphrase); \ No newline at end of file ++ const char *passphrase); From 1c5cdfee91473bc5733479b578d975f1f1eb2044 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Wed, 10 Aug 2016 07:17:12 +0000 Subject: [PATCH 66/80] Set an RPATH on all libraries requiring fortran in binary-dist (#17901) (cherry picked from commit 8615732f097fe3a992803b8d83729bdd315f1725) --- Makefile | 2 +- contrib/fixup-libgfortran.sh | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9d2aa47dd7527..8b20b2e4585e8 100644 --- a/Makefile +++ b/Makefile @@ -450,7 +450,7 @@ endif @$(MAKE) -C $(BUILDROOT) -f $(JULIAHOME)/Makefile install cp $(JULIAHOME)/LICENSE.md $(BUILDROOT)/julia-$(JULIA_COMMIT) ifneq ($(OS), WINNT) - -$(JULIAHOME)/contrib/fixup-libgfortran.sh $(DESTDIR)$(private_libdir) + -$(CUSTOM_LD_LIBRARY_PATH) PATH=$(PATH):$(build_depsbindir) $(JULIAHOME)/contrib/fixup-libgfortran.sh $(DESTDIR)$(private_libdir) endif ifeq ($(OS), Linux) -$(JULIAHOME)/contrib/fixup-libstdc++.sh $(DESTDIR)$(libdir) $(DESTDIR)$(private_libdir) diff --git a/contrib/fixup-libgfortran.sh b/contrib/fixup-libgfortran.sh index a0d52c7f9b5f3..a7ee7025b9a7f 100755 --- a/contrib/fixup-libgfortran.sh +++ b/contrib/fixup-libgfortran.sh @@ -93,3 +93,12 @@ if [ "$UNAME" = "Darwin" ]; then done done fi + +if [ "$UNAME" = "Linux" ]; then + cd $private_libdir + for file in openlibm quadmath gfortran openblas arpack lapack openspecfun; do + for dylib in $(ls lib$file*.so* 2>/dev/null); do + patchelf --set-rpath \$ORIGIN $dylib + done + done +fi From d992f3dd10602ef52d8c297ebb7c7396a579c963 Mon Sep 17 00:00:00 2001 From: Andreas Noack Date: Wed, 10 Aug 2016 03:18:17 -0400 Subject: [PATCH 67/80] Remove redundant uplo argument in chol family. When using Hermitian (#17909) and Symmetric, it is redundant to also have an uplo argument and occasionally, it also gave the wrong result. This can therefore be considered a bugfix. (cherry picked from commit 35df9b806956584043b9ada634124162bccd096e) --- base/deprecated.jl | 3 ++ base/linalg/cholesky.jl | 97 ++++++++++++++++++++++------------------- doc/stdlib/linalg.rst | 12 ++--- test/linalg/cholesky.jl | 10 ++--- 4 files changed, 66 insertions(+), 56 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index ac4eb4e1c5347..002da08644150 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -775,6 +775,9 @@ function transpose(x) return x end +@deprecate cholfact!(A::Base.LinAlg.HermOrSym, uplo::Symbol, ::Type{Val{false}}) cholfact!(A, Val{false}) +@deprecate cholfact!(A::Base.LinAlg.HermOrSym, uplo::Symbol = :U) cholfact!(A) + # During the 0.5 development cycle, do not add any deprecations below this line # To be deprecated in 0.6 diff --git a/base/linalg/cholesky.jl b/base/linalg/cholesky.jl index e961ef09f5a57..ce799f327b782 100644 --- a/base/linalg/cholesky.jl +++ b/base/linalg/cholesky.jl @@ -121,8 +121,10 @@ non_hermitian_error(f) = throw(ArgumentError("matrix is not symmetric/" * # chol!. Destructive methods for computing Cholesky factor of real symmetric or Hermitian # matrix -chol!(A::Hermitian) = _chol!(A.data, UpperTriangular) -chol!{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}) = _chol!(A.data, UpperTriangular) +chol!(A::Hermitian) = + _chol!(A.uplo == 'U' ? A.data : LinAlg.copytri!(A.data, 'L', true), UpperTriangular) +chol!{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}) = + _chol!(A.uplo == 'U' ? A.data : LinAlg.copytri!(A.data, 'L', true), UpperTriangular) function chol!(A::StridedMatrix) ishermitian(A) || non_hermitian_error("chol!") return _chol!(A, UpperTriangular) @@ -135,14 +137,22 @@ end function chol(A::Hermitian) T = promote_type(typeof(chol(one(eltype(A)))), Float32) AA = similar(A, T, size(A)) - copy!(AA, A.data) - chol!(Hermitian(AA)) + if A.uplo == 'U' + copy!(AA, A.data) + else + Base.ccopy!(AA, A.data) + end + chol!(Hermitian(AA, :U)) end function chol{T<:Real,S<:AbstractMatrix}(A::Symmetric{T,S}) TT = promote_type(typeof(chol(one(T))), Float32) AA = similar(A, TT, size(A)) - copy!(AA, A.data) - chol!(Hermitian(AA)) + if A.uplo == 'U' + copy!(AA, A.data) + else + Base.ccopy!(AA, A.data) + end + chol!(Hermitian(AA, :U)) end ## for StridedMatrices, check that matrix is symmetric/Hermitian @@ -170,15 +180,15 @@ chol(x::Number, args...) = _chol!(x, nothing) # cholfact!. Destructive methods for computing Cholesky factorization of real symmetric # or Hermitian matrix ## No pivoting -function cholfact!(A::Hermitian, uplo::Symbol, ::Type{Val{false}}) - if uplo == :U +function cholfact!(A::Hermitian, ::Type{Val{false}}) + if A.uplo == :U Cholesky(_chol!(A.data, UpperTriangular).data, 'U') else Cholesky(_chol!(A.data, LowerTriangular).data, 'L') end end -function cholfact!{T<:Real,S}(A::Symmetric{T,S}, uplo::Symbol, ::Type{Val{false}}) - if uplo == :U +function cholfact!{T<:Real,S}(A::Symmetric{T,S}, ::Type{Val{false}}) + if A.uplo == :U Cholesky(_chol!(A.data, UpperTriangular).data, 'U') else Cholesky(_chol!(A.data, LowerTriangular).data, 'L') @@ -187,7 +197,7 @@ end ### for StridedMatrices, check that matrix is symmetric/Hermitian """ - cholfact!(A, uplo::Symbol, Val{false}) -> Cholesky + cholfact!(A, [uplo::Symbol,] Val{false}) -> Cholesky The same as `cholfact`, but saves space by overwriting the input `A`, instead of creating a copy. An `InexactError` exception is thrown if the factorisation @@ -196,37 +206,36 @@ integer types. """ function cholfact!(A::StridedMatrix, uplo::Symbol, ::Type{Val{false}}) ishermitian(A) || non_hermitian_error("cholfact!") - return cholfact!(Hermitian(A), uplo, Val{false}) + return cholfact!(Hermitian(A, uplo), Val{false}) end ### Default to no pivoting (and storing of upper factor) when not explicit -cholfact!(A::Hermitian, uplo::Symbol = :U) = cholfact!(A, uplo, Val{false}) -cholfact!{T<:Real,S}(A::Symmetric{T,S}, uplo::Symbol = :U) = cholfact!(A, uplo, Val{false}) +cholfact!(A::Hermitian) = cholfact!(A, Val{false}) +cholfact!{T<:Real,S}(A::Symmetric{T,S}) = cholfact!(A, Val{false}) #### for StridedMatrices, check that matrix is symmetric/Hermitian function cholfact!(A::StridedMatrix, uplo::Symbol = :U) ishermitian(A) || non_hermitian_error("cholfact!") - return cholfact!(Hermitian(A), uplo) + return cholfact!(Hermitian(A, uplo)) end ## With pivoting ### BLAS/LAPACK element types function cholfact!{T<:BlasReal,S<:StridedMatrix}(A::RealHermSymComplexHerm{T,S}, - uplo::Symbol, ::Type{Val{true}}; tol = 0.0) - uplochar = char_uplo(uplo) - AA, piv, rank, info = LAPACK.pstrf!(uplochar, A.data, tol) - return CholeskyPivoted{eltype(AA),typeof(AA)}(AA, uplochar, piv, rank, tol, info) + ::Type{Val{true}}; tol = 0.0) + AA, piv, rank, info = LAPACK.pstrf!(A.uplo, A.data, tol) + return CholeskyPivoted{eltype(AA),typeof(AA)}(AA, A.uplo, piv, rank, tol, info) end ### Non BLAS/LAPACK element types (generic). Since generic fallback for pivoted Cholesky ### is not implemented yet we throw an error -cholfact!{T<:Real,S}(A::RealHermSymComplexHerm{T,S}, uplo::Symbol, ::Type{Val{true}}; +cholfact!{T<:Real,S}(A::RealHermSymComplexHerm{T,S}, ::Type{Val{true}}; tol = 0.0) = throw(ArgumentError("generic pivoted Cholesky factorization is not implemented yet")) ### for StridedMatrices, check that matrix is symmetric/Hermitian """ - cholfact!(A, uplo::Symbol, Val{true}; tol = 0.0) -> CholeskyPivoted + cholfact!(A, [uplo::Symbol,] Val{true}; tol = 0.0) -> CholeskyPivoted The same as `cholfact`, but saves space by overwriting the input `A`, instead of creating a copy. An `InexactError` exception is thrown if the @@ -235,26 +244,25 @@ e.g. for integer types. """ function cholfact!(A::StridedMatrix, uplo::Symbol, ::Type{Val{true}}; tol = 0.0) ishermitian(A) || non_hermitian_error("cholfact!") - return cholfact!(Hermitian(A), uplo, Val{true}; tol = tol) + return cholfact!(Hermitian(A, uplo), Val{true}; tol = tol) end - - # cholfact. Non-destructive methods for computing Cholesky factorization of real symmetric # or Hermitian matrix ## No pivoting -cholfact(A::Hermitian, uplo::Symbol, ::Type{Val{false}}) = - cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), uplo, Val{false}) -cholfact{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}, uplo::Symbol, ::Type{Val{false}}) = - cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), uplo, Val{false}) +cholfact(A::Hermitian, ::Type{Val{false}}) = + cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), Val{false}) +cholfact{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}, ::Type{Val{false}}) = + cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), Val{false}) ### for StridedMatrices, check that matrix is symmetric/Hermitian """ - cholfact(A, uplo::Symbol, Val{false}) -> Cholesky + cholfact(A, [uplo::Symbol,] Val{false}) -> Cholesky Compute the Cholesky factorization of a dense symmetric positive definite matrix `A` -and return a `Cholesky` factorization. -The `uplo` argument may be `:L` for using the lower part or `:U` for the upper part of `A`. +and return a `Cholesky` factorization. The matrix `A` can either be a `Symmetric` or `Hermitian` +`StridedMatrix` or a *perfectly* symmetric or Hermitian `StridedMatrix`. In the latter case, +the optional argument `uplo` may be `:L` for using the lower part or `:U` for the upper part of `A`. The default is to use `:U`. The triangular Cholesky factor can be obtained from the factorization `F` with: `F[:L]` and `F[:U]`. The following functions are available for `Cholesky` objects: `size`, `\\`, `inv`, `det`. @@ -262,36 +270,35 @@ A `PosDefException` exception is thrown in case the matrix is not positive defin """ function cholfact(A::StridedMatrix, uplo::Symbol, ::Type{Val{false}}) ishermitian(A) || non_hermitian_error("cholfact") - return cholfact(Hermitian(A), uplo, Val{false}) + return cholfact(Hermitian(A, uplo), Val{false}) end ### Default to no pivoting (and storing of upper factor) when not explicit -cholfact(A::Hermitian, uplo::Symbol = :U) = cholfact(A, uplo, Val{false}) -cholfact{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}, uplo::Symbol = :U) = - cholfact(A, uplo, Val{false}) +cholfact(A::Hermitian) = cholfact(A, Val{false}) +cholfact{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}) = cholfact(A, Val{false}) #### for StridedMatrices, check that matrix is symmetric/Hermitian function cholfact(A::StridedMatrix, uplo::Symbol = :U) ishermitian(A) || non_hermitian_error("cholfact") - return cholfact(Hermitian(A), uplo) + return cholfact(Hermitian(A, uplo)) end ## With pivoting -cholfact(A::Hermitian, uplo::Symbol, ::Type{Val{true}}; tol = 0.0) = +cholfact(A::Hermitian, ::Type{Val{true}}; tol = 0.0) = cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), - uplo, Val{true}; tol = tol) -cholfact{T<:Real,S<:StridedMatrix}(A::RealHermSymComplexHerm{T,S}, uplo::Symbol, - ::Type{Val{true}}; tol = 0.0) = + Val{true}; tol = tol) +cholfact{T<:Real,S<:StridedMatrix}(A::RealHermSymComplexHerm{T,S}, ::Type{Val{true}}; tol = 0.0) = cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), - uplo, Val{true}; tol = tol) + Val{true}; tol = tol) ### for StridedMatrices, check that matrix is symmetric/Hermitian """ - cholfact(A, uplo::Symbol, Val{true}; tol = 0.0) -> CholeskyPivoted + cholfact(A, [uplo::Symbol,] Val{true}; tol = 0.0) -> CholeskyPivoted Compute the pivoted Cholesky factorization of a dense symmetric positive semi-definite matrix `A` -and return a `CholeskyPivoted` factorization. -The `uplo` argument may be `:L` for using the lower part or `:U` for the upper part of `A`. +and return a `CholeskyPivoted` factorization. The matrix `A` can either be a `Symmetric` or `Hermitian` +`StridedMatrix` or a *perfectly* symmetric or Hermitian `StridedMatrix`. In the latter case, +the optional argument `uplo` may be `:L` for using the lower part or `:U` for the upper part of `A`. The default is to use `:U`. The triangular Cholesky factor can be obtained from the factorization `F` with: `F[:L]` and `F[:U]`. The following functions are available for `PivotedCholesky` objects: `size`, `\\`, `inv`, `det`, and `rank`. @@ -300,7 +307,7 @@ For negative values, the tolerance is the machine precision. """ function cholfact(A::StridedMatrix, uplo::Symbol, ::Type{Val{true}}; tol = 0.0) ishermitian(A) || non_hermitian_error("cholfact") - return cholfact(Hermitian(A), uplo, Val{true}; tol = tol) + return cholfact(Hermitian(A, uplo), Val{true}; tol = tol) end ## Number diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index 2bd41341c74fb..349869b5a5214 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -308,17 +308,17 @@ Linear algebra functions in Julia are largely implemented by calling functions f Compute the square root of a non-negative number ``x``\ . -.. function:: cholfact(A, uplo::Symbol, Val{false}) -> Cholesky +.. function:: cholfact(A, [uplo::Symbol,] Val{false}) -> Cholesky .. Docstring generated from Julia source - Compute the Cholesky factorization of a dense symmetric positive definite matrix ``A`` and return a ``Cholesky`` factorization. The ``uplo`` argument may be ``:L`` for using the lower part or ``:U`` for the upper part of ``A``\ . The default is to use ``:U``\ . The triangular Cholesky factor can be obtained from the factorization ``F`` with: ``F[:L]`` and ``F[:U]``\ . The following functions are available for ``Cholesky`` objects: ``size``\ , ``\``\ , ``inv``\ , ``det``\ . A ``PosDefException`` exception is thrown in case the matrix is not positive definite. + Compute the Cholesky factorization of a dense symmetric positive definite matrix ``A`` and return a ``Cholesky`` factorization. The matrix ``A`` can either be a ``Symmetric`` or ``Hermitian`` ``StridedMatrix`` or a *perfectly* symmetric or Hermitian ``StridedMatrix``\ . In the latter case, the optional argument ``uplo`` may be ``:L`` for using the lower part or ``:U`` for the upper part of ``A``\ . The default is to use ``:U``\ . The triangular Cholesky factor can be obtained from the factorization ``F`` with: ``F[:L]`` and ``F[:U]``\ . The following functions are available for ``Cholesky`` objects: ``size``\ , ``\``\ , ``inv``\ , ``det``\ . A ``PosDefException`` exception is thrown in case the matrix is not positive definite. -.. function:: cholfact(A, uplo::Symbol, Val{true}; tol = 0.0) -> CholeskyPivoted +.. function:: cholfact(A, [uplo::Symbol,] Val{true}; tol = 0.0) -> CholeskyPivoted .. Docstring generated from Julia source - Compute the pivoted Cholesky factorization of a dense symmetric positive semi-definite matrix ``A`` and return a ``CholeskyPivoted`` factorization. The ``uplo`` argument may be ``:L`` for using the lower part or ``:U`` for the upper part of ``A``\ . The default is to use ``:U``\ . The triangular Cholesky factor can be obtained from the factorization ``F`` with: ``F[:L]`` and ``F[:U]``\ . The following functions are available for ``PivotedCholesky`` objects: ``size``\ , ``\``\ , ``inv``\ , ``det``\ , and ``rank``\ . The argument ``tol`` determines the tolerance for determining the rank. For negative values, the tolerance is the machine precision. + Compute the pivoted Cholesky factorization of a dense symmetric positive semi-definite matrix ``A`` and return a ``CholeskyPivoted`` factorization. The matrix ``A`` can either be a ``Symmetric`` or ``Hermitian`` ``StridedMatrix`` or a *perfectly* symmetric or Hermitian ``StridedMatrix``\ . In the latter case, the optional argument ``uplo`` may be ``:L`` for using the lower part or ``:U`` for the upper part of ``A``\ . The default is to use ``:U``\ . The triangular Cholesky factor can be obtained from the factorization ``F`` with: ``F[:L]`` and ``F[:U]``\ . The following functions are available for ``PivotedCholesky`` objects: ``size``\ , ``\``\ , ``inv``\ , ``det``\ , and ``rank``\ . The argument ``tol`` determines the tolerance for determining the rank. For negative values, the tolerance is the machine precision. .. function:: cholfact(A; shift = 0.0, perm = Int[]) -> CHOLMOD.Factor @@ -344,13 +344,13 @@ Linear algebra functions in Julia are largely implemented by calling functions f This method uses the CHOLMOD library from SuiteSparse, which only supports doubles or complex doubles. Input matrices not of those element types will be converted to ``SparseMatrixCSC{Float64}`` or ``SparseMatrixCSC{Complex128}`` as appropriate. -.. function:: cholfact!(A, uplo::Symbol, Val{false}) -> Cholesky +.. function:: cholfact!(A, [uplo::Symbol,] Val{false}) -> Cholesky .. Docstring generated from Julia source The same as ``cholfact``\ , but saves space by overwriting the input ``A``\ , instead of creating a copy. An ``InexactError`` exception is thrown if the factorisation produces a number not representable by the element type of ``A``\ , e.g. for integer types. -.. function:: cholfact!(A, uplo::Symbol, Val{true}; tol = 0.0) -> CholeskyPivoted +.. function:: cholfact!(A, [uplo::Symbol,] Val{true}; tol = 0.0) -> CholeskyPivoted .. Docstring generated from Julia source diff --git a/test/linalg/cholesky.jl b/test/linalg/cholesky.jl index 00e4696b4f12b..bb0657c799838 100644 --- a/test/linalg/cholesky.jl +++ b/test/linalg/cholesky.jl @@ -74,15 +74,15 @@ for eltya in (Float32, Float64, Complex64, Complex128, BigFloat, Int) @test full(lapd) ≈ apd l = lapd[:L] @test l*l' ≈ apd - @test triu(capd.factors) ≈ lapd[:U] - @test tril(lapd.factors) ≈ capd[:L] + @test capd[:U] ≈ lapd[:U] + @test lapd[:L] ≈ capd[:L] if eltya <: Real capds = cholfact(apds) - lapds = cholfact(apds, :L) + lapds = cholfact(full(apds), :L) ls = lapds[:L] @test ls*ls' ≈ apd - @test triu(capds.factors) ≈ lapds[:U] - @test tril(lapds.factors) ≈ capds[:L] + @test capds[:U] ≈ lapds[:U] + @test lapds[:L] ≈ capds[:L] end #pivoted upper Cholesky From b311b03479ac8c3d1e636f9b538468ee5063464b Mon Sep 17 00:00:00 2001 From: Andreas Noack Date: Wed, 10 Aug 2016 03:20:14 -0400 Subject: [PATCH 68/80] Define conj and conj! for for Symmetric and Hermitian. (#17827) Fixes #17780 (cherry picked from commit 3f6b2b26a2d4ccef77f0b54ae664e5dcae157db1) --- base/linalg/symmetric.jl | 3 +++ test/linalg/symmetric.jl | 14 ++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/base/linalg/symmetric.jl b/base/linalg/symmetric.jl index 0a52b6041adbf..d1da83d347d15 100644 --- a/base/linalg/symmetric.jl +++ b/base/linalg/symmetric.jl @@ -114,6 +114,9 @@ end ctranspose(A::Hermitian) = A trace(A::Hermitian) = real(trace(A.data)) +Base.conj(A::HermOrSym) = typeof(A)(conj(A.data), A.uplo) +Base.conj!(A::HermOrSym) = typeof(A)(conj!(A.data), A.uplo) + #tril/triu function tril(A::Hermitian, k::Integer=0) if A.uplo == 'U' && k <= 0 diff --git a/test/linalg/symmetric.jl b/test/linalg/symmetric.jl index 3a33105b5372f..55f815f500fcb 100644 --- a/test/linalg/symmetric.jl +++ b/test/linalg/symmetric.jl @@ -236,3 +236,17 @@ let A = Symmetric(randn(5,5)) B = -A @test A + B ≈ zeros(5,5) end + +# 17780 +let a = randn(2,2) + a = a'a + b = complex(a,a) + c = Symmetric(b) + @test conj(c) == conj(Array(c)) + cc = copy(c) + @test conj!(c) == conj(Array(cc)) + c = Hermitian(b + b') + @test conj(c) == conj(Array(c)) + cc = copy(c) + @test conj!(c) == conj(Array(c)) +end From 1bd4fbbff1da27d1c7c6118cbb8ee022e11330b5 Mon Sep 17 00:00:00 2001 From: Ranjan Anantharaman Date: Wed, 10 Aug 2016 15:22:05 +0530 Subject: [PATCH 69/80] Add cross-release compatibility note to CONTRIBUTING.md (#17911) * Add cross-release compatibility note to CONTRIBUTING.md * tag -> specify (cherry picked from commit dcffb4a3b2bde018851d70450cbc9d7da2ca3988) --- CONTRIBUTING.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index adf70e7efbfee..dbd0fe9179efa 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -60,6 +60,22 @@ Julia has a built-in [package manager](https://github.com/JuliaLang/METADATA.jl) For developers who need to wrap C libraries so that they can be called from Julia, the [Clang.jl](https://github.com/ihnorton/Clang.jl) package can help generate the wrappers automatically from the C header files. +### Package Compatibility Across Releases + +Sometimes, you might find that while your package works +on the current release, it might not work on the upcoming release or nightly. +This is due to the fact that some Julia functions (after some discussion) +could be deprecated or removed altogether. This may cause your package to break or +throw a number of deprecation warnings on usage. Therefore it is highly recommended +to port your package to latest Julia release. + +However, porting a package to the latest release may cause the package to break on +earlier Julia releases. To maintain compatibility across releases, use +[`Compat.jl`](https://github.com/JuliaLang/Compat.jl/). Find the fix for your package +from the README, and specify the minimum version of Compat that provides the fix +in your REQUIRE file. To tag the correct minimum version, refer to +[this guide](https://github.com/JuliaLang/Compat.jl/#tagging-the-correct-minimum-version-of-compat). + ### Writing tests There are never enough tests. Track [code coverage at Coveralls](https://coveralls.io/r/JuliaLang/julia), and help improve it. From 058dcea24c53f1bd747618c5aae86a0752eefbf5 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Wed, 10 Aug 2016 02:57:07 -0700 Subject: [PATCH 70/80] reword second tag -> find in CONTRIBUTING.md ref #17911 (cherry picked from commit 091c2fd6485fd042f17651b1d869446c6b640db8) --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dbd0fe9179efa..a2040fbf5ef75 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -73,7 +73,7 @@ However, porting a package to the latest release may cause the package to break earlier Julia releases. To maintain compatibility across releases, use [`Compat.jl`](https://github.com/JuliaLang/Compat.jl/). Find the fix for your package from the README, and specify the minimum version of Compat that provides the fix -in your REQUIRE file. To tag the correct minimum version, refer to +in your REQUIRE file. To find the correct minimum version, refer to [this guide](https://github.com/JuliaLang/Compat.jl/#tagging-the-correct-minimum-version-of-compat). ### Writing tests From c3cd268558a3e54ddbf25037b2116e8b45f21253 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 10 Aug 2016 13:10:12 +0200 Subject: [PATCH 71/80] improve heuristic for when to refresh line in the REPL (#17868) * improve heuristic for when to refresh line * fix trailing whitespace... * use indent when not on first line * remove unused argument and change local var to not share name with function (cherry picked from commit f84b94ccac49c0a505b639122bb044069659f6d8) --- base/LineEdit.jl | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/base/LineEdit.jl b/base/LineEdit.jl index a0a45e63cc73b..9298c4dee2bc2 100644 --- a/base/LineEdit.jl +++ b/base/LineEdit.jl @@ -447,10 +447,19 @@ function edit_replace(s, from, to, str) end function edit_insert(s::PromptState, c) + buf = s.input_buffer + function line_size() + p = position(buf) + seek(buf, rsearch(buf.data, '\n', p)) + ls = p - position(buf) + seek(buf, p) + return ls + end str = string(c) - edit_insert(s.input_buffer, str) - if !('\n' in str) && eof(s.input_buffer) && - ((position(s.input_buffer) + sizeof(s.p.prompt) + sizeof(str) - 1) < width(terminal(s))) + edit_insert(buf, str) + offset = s.ias.curs_row == 1 ? sizeof(s.p.prompt) : s.indent + if !('\n' in str) && eof(buf) && + ((line_size() + offset + sizeof(str) - 1) < width(terminal(s))) # Avoid full update when appending characters to the end # and an update of curs_row isn't necessary (conservatively estimated) write(terminal(s), str) From 03cf9272360a1410c3d48cfb2fb387e5b9c93663 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Wed, 10 Aug 2016 15:40:48 -0400 Subject: [PATCH 72/80] fix missing line break in precompile error message (cherry picked from commit 8dfe9c83edeff215c47729d3677e5775ab804af3) ref #17947 --- src/dump.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dump.c b/src/dump.c index d46fb1b624d7a..cfd8cd7ba48ea 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1752,7 +1752,7 @@ static int read_verify_mod_list(ios_t *s) if (m->uuid != uuid) { jl_printf(JL_STDERR, "WARNING: Module %s uuid did not match cache file\n" - " This is likely because module %s does not support" + " This is likely because module %s does not support\n" " precompilation but is imported by a module that does.\n", name, name); return 0; From 826f27b101086ea220689a1691ef6553b5c72a4e Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Wed, 10 Aug 2016 17:39:13 -0400 Subject: [PATCH 73/80] fix BigInt `ndigits` test See #16766. `ndigits(big(0),1)` doesn't raise an error, but we were sometimes (randomly, rarely) testing that it does. This change makes the test reliable. The issue of how this case should actually behave is still open. (cherry picked from commit dd09d19f6545a87cef384378d6956ce48c04d9a5) ref #17952 --- test/bigint.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/bigint.jl b/test/bigint.jl index 999b0c42cf6dd..0478b28f8b286 100644 --- a/test/bigint.jl +++ b/test/bigint.jl @@ -278,7 +278,11 @@ ndigits_mismatch(n) = ndigits(n) != ndigits(BigInt(n)) ndigits(rand(big(-999:999)), rand(63:typemax(Int))) ndigits(rand(big(-999:999)), big(2)^rand(2:999)) -@test_throws DomainError ndigits(rand(big(-999:999)), rand(typemin(Int):1)) +for i in big([-20:-1;1:20]) + for b in -10:1 + @test_throws DomainError ndigits(i, b) + end +end # conversion from float @test BigInt(2.0) == BigInt(2.0f0) == BigInt(big(2.0)) == 2 From aa59b4201e702ff6bacbf77d1e140bc4588741b9 Mon Sep 17 00:00:00 2001 From: Mus M Date: Thu, 11 Aug 2016 00:51:09 -0400 Subject: [PATCH 74/80] fix output of getaddrinfo in networking and streams (#17931) fix output of getaddrinfo in networking and streams documentation for v0.5. (cherry picked from commit 068aa9d3aaae6c71b92845f8ee39feb139de0761) --- doc/manual/networking-and-streams.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/networking-and-streams.rst b/doc/manual/networking-and-streams.rst index cbaa39eba5985..de39508cdd6a6 100644 --- a/doc/manual/networking-and-streams.rst +++ b/doc/manual/networking-and-streams.rst @@ -260,5 +260,5 @@ allows you to do things like:: At the base of this functionality is :func:`getaddrinfo`, which will do the appropriate address resolution:: julia> getaddrinfo("google.com") - IPv4(74.125.226.225) + ip"74.125.226.225" From 84ad7c720cb0aaaecb226388eb9c00a5a2ba2041 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Thu, 11 Aug 2016 02:10:44 -0400 Subject: [PATCH 75/80] Fix doctest line numbers --- base/array.jl | 2 +- base/linalg/eigen.jl | 10 ++++++---- doc/manual/mathematical-operations.rst | 2 +- doc/stdlib/collections.rst | 2 +- doc/stdlib/linalg.rst | 10 ++++++---- doc/stdlib/pkg.rst | 2 +- 6 files changed, 16 insertions(+), 12 deletions(-) diff --git a/base/array.jl b/base/array.jl index a24a7773477a7..4eba2f531ad78 100644 --- a/base/array.jl +++ b/base/array.jl @@ -594,7 +594,7 @@ julia> deleteat!([6, 5, 4, 3, 2, 1], 1:2:5) julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) ERROR: ArgumentError: indices must be unique and sorted - in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:576 + in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:611 ... ``` """ diff --git a/base/linalg/eigen.jl b/base/linalg/eigen.jl index 93e9468fd24a7..21067ac34e024 100644 --- a/base/linalg/eigen.jl +++ b/base/linalg/eigen.jl @@ -175,8 +175,9 @@ julia> A = [0 im; -1 0] julia> eigmax(A) ERROR: DomainError: - in #eigmax#30(::Bool, ::Bool, ::Function, ::Array{Complex{Int64},2}) at ./linalg/eigen.jl:108 - in eigmax(::Array{Complex{Int64},2}) at ./linalg/eigen.jl:106 + in #eigmax#30(::Bool, ::Bool, ::Function, ::Array{Complex{Int64},2}) at ./linalg/eigen.jl:186 + in eigmax(::Array{Complex{Int64},2}) at ./linalg/eigen.jl:184 + ... ``` """ function eigmax(A::Union{Number, StridedMatrix}; permute::Bool=true, scale::Bool=true) @@ -214,8 +215,9 @@ julia> A = [0 im; -1 0] julia> eigmin(A) ERROR: DomainError: - in #eigmin#31(::Bool, ::Bool, ::Function, ::Array{Complex{Int64},2}) at ./linalg/eigen.jl:115 - in eigmin(::Array{Complex{Int64},2}) at ./linalg/eigen.jl:113 + in #eigmin#31(::Bool, ::Bool, ::Function, ::Array{Complex{Int64},2}) at ./linalg/eigen.jl:226 + in eigmin(::Array{Complex{Int64},2}) at ./linalg/eigen.jl:224 + ... ``` """ function eigmin(A::Union{Number, StridedMatrix}; permute::Bool=true, scale::Bool=true) diff --git a/doc/manual/mathematical-operations.rst b/doc/manual/mathematical-operations.rst index 36fba25e8c46f..c7925dea544e6 100644 --- a/doc/manual/mathematical-operations.rst +++ b/doc/manual/mathematical-operations.rst @@ -427,7 +427,7 @@ The following examples show the different forms. julia> round(Int8,127.6) ERROR: InexactError() - in trunc(::Type{Int8}, ::Float64) at ./float.jl:458 + in trunc(::Type{Int8}, ::Float64) at ./float.jl:456 in round(::Type{Int8}, ::Float64) at ./float.jl:211 ... diff --git a/doc/stdlib/collections.rst b/doc/stdlib/collections.rst index a8b4285b754c4..59abf1b78aba2 100644 --- a/doc/stdlib/collections.rst +++ b/doc/stdlib/collections.rst @@ -1379,7 +1379,7 @@ Dequeues julia> deleteat!([6, 5, 4, 3, 2, 1], (2, 2)) ERROR: ArgumentError: indices must be unique and sorted - in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:576 + in deleteat!(::Array{Int64,1}, ::Tuple{Int64,Int64}) at ./array.jl:611 ... .. function:: splice!(collection, index, [replacement]) -> item diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index 349869b5a5214..ff1a4808cd8d5 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -651,8 +651,9 @@ Linear algebra functions in Julia are largely implemented by calling functions f julia> eigmax(A) ERROR: DomainError: - in #eigmax#30(::Bool, ::Bool, ::Function, ::Array{Complex{Int64},2}) at ./linalg/eigen.jl:108 - in eigmax(::Array{Complex{Int64},2}) at ./linalg/eigen.jl:106 + in #eigmax#30(::Bool, ::Bool, ::Function, ::Array{Complex{Int64},2}) at ./linalg/eigen.jl:186 + in eigmax(::Array{Complex{Int64},2}) at ./linalg/eigen.jl:184 + ... .. function:: eigmin(A; permute::Bool=true, scale::Bool=true) @@ -677,8 +678,9 @@ Linear algebra functions in Julia are largely implemented by calling functions f julia> eigmin(A) ERROR: DomainError: - in #eigmin#31(::Bool, ::Bool, ::Function, ::Array{Complex{Int64},2}) at ./linalg/eigen.jl:115 - in eigmin(::Array{Complex{Int64},2}) at ./linalg/eigen.jl:113 + in #eigmin#31(::Bool, ::Bool, ::Function, ::Array{Complex{Int64},2}) at ./linalg/eigen.jl:226 + in eigmin(::Array{Complex{Int64},2}) at ./linalg/eigen.jl:224 + ... .. function:: eigvecs(A, [eigvals,][permute=true,][scale=true]) -> Matrix diff --git a/doc/stdlib/pkg.rst b/doc/stdlib/pkg.rst index 348505145fbb7..6df17bb8108a0 100644 --- a/doc/stdlib/pkg.rst +++ b/doc/stdlib/pkg.rst @@ -13,7 +13,7 @@ Functions for package development (e.g. ``tag``, ``publish``, etc.) have been mo .. Docstring generated from Julia source - Returns the absolute path of the package directory. This defaults to ``joinpath(homedir(),".julia","v$(VERSION.major).$(VERSION.minor)")`` on all platforms (i.e. ``~/.julia/v0.6`` in UNIX shell syntax). If the ``JULIA_PKGDIR`` environment variable is set, then that path is used in the returned value as ``joinpath(ENV["JULIA_PKGDIR"],"v$(VERSION.major).$(VERSION.minor)")``\ . If ``JULIA_PKGDIR`` is a relative path, it is interpreted relative to whatever the current working directory is. + Returns the absolute path of the package directory. This defaults to ``joinpath(homedir(),".julia","v$(VERSION.major).$(VERSION.minor)")`` on all platforms (i.e. ``~/.julia/v0.5`` in UNIX shell syntax). If the ``JULIA_PKGDIR`` environment variable is set, then that path is used in the returned value as ``joinpath(ENV["JULIA_PKGDIR"],"v$(VERSION.major).$(VERSION.minor)")``\ . If ``JULIA_PKGDIR`` is a relative path, it is interpreted relative to whatever the current working directory is. .. function:: dir(names...) -> AbstractString From 799af802daa5ae14713ae3f85ca8a4fab5b1d875 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Thu, 11 Aug 2016 03:11:52 -0400 Subject: [PATCH 76/80] license header for weakkeydict.jl --- base/weakkeydict.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/base/weakkeydict.jl b/base/weakkeydict.jl index 6485c83af38e4..d3a7416924abc 100644 --- a/base/weakkeydict.jl +++ b/base/weakkeydict.jl @@ -1,3 +1,5 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + # weak key dictionaries type WeakKeyDict{K,V} <: Associative{K,V} From a228a7aa6a9a16ac1d2ec53fa0aae5ab9070b208 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 12 Aug 2016 02:12:35 +0000 Subject: [PATCH 77/80] Fix typo in libgit2 credentials callback (#17975) (cherry picked from commit 1d5a4272f6563e3cce3ca5e8746c2eafa0a86dc9) --- base/libgit2/callbacks.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index 91a0b8d4ca42b..fd2580da27ee7 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -121,8 +121,8 @@ function authenticate_ssh(creds::SSHCredentials, libgit2credptr::Ptr{Ptr{Void}}, end passdef end - (creds.user != username) || (creds.pass != userpass) || - (creds.prvkey != privatekey) || (creds.pubkey != publickey) && reset!(creds) + ((creds.user != username) || (creds.pass != passphrase) || + (creds.prvkey != privatekey) || (creds.pubkey != publickey)) && reset!(creds) creds.user = username # save credentials creds.prvkey = privatekey # save credentials From 0eabf45486b2cf5f59939b51433b5669ce4ab11d Mon Sep 17 00:00:00 2001 From: Andreas Noack Date: Thu, 11 Aug 2016 21:17:18 -0400 Subject: [PATCH 78/80] Revert "Remove redundant uplo argument in chol family. When using Hermitian (#17909)" This reverts commit d992f3dd10602ef52d8c297ebb7c7396a579c963. (cherry picked from commit 62b55ce011cba72c96a1dc5f178ccd27c50c6e6d) ref #17985 --- base/deprecated.jl | 3 -- base/linalg/cholesky.jl | 97 +++++++++++++++++++---------------------- doc/stdlib/linalg.rst | 12 ++--- test/linalg/cholesky.jl | 10 ++--- 4 files changed, 56 insertions(+), 66 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index 002da08644150..ac4eb4e1c5347 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -775,9 +775,6 @@ function transpose(x) return x end -@deprecate cholfact!(A::Base.LinAlg.HermOrSym, uplo::Symbol, ::Type{Val{false}}) cholfact!(A, Val{false}) -@deprecate cholfact!(A::Base.LinAlg.HermOrSym, uplo::Symbol = :U) cholfact!(A) - # During the 0.5 development cycle, do not add any deprecations below this line # To be deprecated in 0.6 diff --git a/base/linalg/cholesky.jl b/base/linalg/cholesky.jl index ce799f327b782..e961ef09f5a57 100644 --- a/base/linalg/cholesky.jl +++ b/base/linalg/cholesky.jl @@ -121,10 +121,8 @@ non_hermitian_error(f) = throw(ArgumentError("matrix is not symmetric/" * # chol!. Destructive methods for computing Cholesky factor of real symmetric or Hermitian # matrix -chol!(A::Hermitian) = - _chol!(A.uplo == 'U' ? A.data : LinAlg.copytri!(A.data, 'L', true), UpperTriangular) -chol!{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}) = - _chol!(A.uplo == 'U' ? A.data : LinAlg.copytri!(A.data, 'L', true), UpperTriangular) +chol!(A::Hermitian) = _chol!(A.data, UpperTriangular) +chol!{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}) = _chol!(A.data, UpperTriangular) function chol!(A::StridedMatrix) ishermitian(A) || non_hermitian_error("chol!") return _chol!(A, UpperTriangular) @@ -137,22 +135,14 @@ end function chol(A::Hermitian) T = promote_type(typeof(chol(one(eltype(A)))), Float32) AA = similar(A, T, size(A)) - if A.uplo == 'U' - copy!(AA, A.data) - else - Base.ccopy!(AA, A.data) - end - chol!(Hermitian(AA, :U)) + copy!(AA, A.data) + chol!(Hermitian(AA)) end function chol{T<:Real,S<:AbstractMatrix}(A::Symmetric{T,S}) TT = promote_type(typeof(chol(one(T))), Float32) AA = similar(A, TT, size(A)) - if A.uplo == 'U' - copy!(AA, A.data) - else - Base.ccopy!(AA, A.data) - end - chol!(Hermitian(AA, :U)) + copy!(AA, A.data) + chol!(Hermitian(AA)) end ## for StridedMatrices, check that matrix is symmetric/Hermitian @@ -180,15 +170,15 @@ chol(x::Number, args...) = _chol!(x, nothing) # cholfact!. Destructive methods for computing Cholesky factorization of real symmetric # or Hermitian matrix ## No pivoting -function cholfact!(A::Hermitian, ::Type{Val{false}}) - if A.uplo == :U +function cholfact!(A::Hermitian, uplo::Symbol, ::Type{Val{false}}) + if uplo == :U Cholesky(_chol!(A.data, UpperTriangular).data, 'U') else Cholesky(_chol!(A.data, LowerTriangular).data, 'L') end end -function cholfact!{T<:Real,S}(A::Symmetric{T,S}, ::Type{Val{false}}) - if A.uplo == :U +function cholfact!{T<:Real,S}(A::Symmetric{T,S}, uplo::Symbol, ::Type{Val{false}}) + if uplo == :U Cholesky(_chol!(A.data, UpperTriangular).data, 'U') else Cholesky(_chol!(A.data, LowerTriangular).data, 'L') @@ -197,7 +187,7 @@ end ### for StridedMatrices, check that matrix is symmetric/Hermitian """ - cholfact!(A, [uplo::Symbol,] Val{false}) -> Cholesky + cholfact!(A, uplo::Symbol, Val{false}) -> Cholesky The same as `cholfact`, but saves space by overwriting the input `A`, instead of creating a copy. An `InexactError` exception is thrown if the factorisation @@ -206,36 +196,37 @@ integer types. """ function cholfact!(A::StridedMatrix, uplo::Symbol, ::Type{Val{false}}) ishermitian(A) || non_hermitian_error("cholfact!") - return cholfact!(Hermitian(A, uplo), Val{false}) + return cholfact!(Hermitian(A), uplo, Val{false}) end ### Default to no pivoting (and storing of upper factor) when not explicit -cholfact!(A::Hermitian) = cholfact!(A, Val{false}) -cholfact!{T<:Real,S}(A::Symmetric{T,S}) = cholfact!(A, Val{false}) +cholfact!(A::Hermitian, uplo::Symbol = :U) = cholfact!(A, uplo, Val{false}) +cholfact!{T<:Real,S}(A::Symmetric{T,S}, uplo::Symbol = :U) = cholfact!(A, uplo, Val{false}) #### for StridedMatrices, check that matrix is symmetric/Hermitian function cholfact!(A::StridedMatrix, uplo::Symbol = :U) ishermitian(A) || non_hermitian_error("cholfact!") - return cholfact!(Hermitian(A, uplo)) + return cholfact!(Hermitian(A), uplo) end ## With pivoting ### BLAS/LAPACK element types function cholfact!{T<:BlasReal,S<:StridedMatrix}(A::RealHermSymComplexHerm{T,S}, - ::Type{Val{true}}; tol = 0.0) - AA, piv, rank, info = LAPACK.pstrf!(A.uplo, A.data, tol) - return CholeskyPivoted{eltype(AA),typeof(AA)}(AA, A.uplo, piv, rank, tol, info) + uplo::Symbol, ::Type{Val{true}}; tol = 0.0) + uplochar = char_uplo(uplo) + AA, piv, rank, info = LAPACK.pstrf!(uplochar, A.data, tol) + return CholeskyPivoted{eltype(AA),typeof(AA)}(AA, uplochar, piv, rank, tol, info) end ### Non BLAS/LAPACK element types (generic). Since generic fallback for pivoted Cholesky ### is not implemented yet we throw an error -cholfact!{T<:Real,S}(A::RealHermSymComplexHerm{T,S}, ::Type{Val{true}}; +cholfact!{T<:Real,S}(A::RealHermSymComplexHerm{T,S}, uplo::Symbol, ::Type{Val{true}}; tol = 0.0) = throw(ArgumentError("generic pivoted Cholesky factorization is not implemented yet")) ### for StridedMatrices, check that matrix is symmetric/Hermitian """ - cholfact!(A, [uplo::Symbol,] Val{true}; tol = 0.0) -> CholeskyPivoted + cholfact!(A, uplo::Symbol, Val{true}; tol = 0.0) -> CholeskyPivoted The same as `cholfact`, but saves space by overwriting the input `A`, instead of creating a copy. An `InexactError` exception is thrown if the @@ -244,25 +235,26 @@ e.g. for integer types. """ function cholfact!(A::StridedMatrix, uplo::Symbol, ::Type{Val{true}}; tol = 0.0) ishermitian(A) || non_hermitian_error("cholfact!") - return cholfact!(Hermitian(A, uplo), Val{true}; tol = tol) + return cholfact!(Hermitian(A), uplo, Val{true}; tol = tol) end + + # cholfact. Non-destructive methods for computing Cholesky factorization of real symmetric # or Hermitian matrix ## No pivoting -cholfact(A::Hermitian, ::Type{Val{false}}) = - cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), Val{false}) -cholfact{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}, ::Type{Val{false}}) = - cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), Val{false}) +cholfact(A::Hermitian, uplo::Symbol, ::Type{Val{false}}) = + cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), uplo, Val{false}) +cholfact{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}, uplo::Symbol, ::Type{Val{false}}) = + cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), uplo, Val{false}) ### for StridedMatrices, check that matrix is symmetric/Hermitian """ - cholfact(A, [uplo::Symbol,] Val{false}) -> Cholesky + cholfact(A, uplo::Symbol, Val{false}) -> Cholesky Compute the Cholesky factorization of a dense symmetric positive definite matrix `A` -and return a `Cholesky` factorization. The matrix `A` can either be a `Symmetric` or `Hermitian` -`StridedMatrix` or a *perfectly* symmetric or Hermitian `StridedMatrix`. In the latter case, -the optional argument `uplo` may be `:L` for using the lower part or `:U` for the upper part of `A`. +and return a `Cholesky` factorization. +The `uplo` argument may be `:L` for using the lower part or `:U` for the upper part of `A`. The default is to use `:U`. The triangular Cholesky factor can be obtained from the factorization `F` with: `F[:L]` and `F[:U]`. The following functions are available for `Cholesky` objects: `size`, `\\`, `inv`, `det`. @@ -270,35 +262,36 @@ A `PosDefException` exception is thrown in case the matrix is not positive defin """ function cholfact(A::StridedMatrix, uplo::Symbol, ::Type{Val{false}}) ishermitian(A) || non_hermitian_error("cholfact") - return cholfact(Hermitian(A, uplo), Val{false}) + return cholfact(Hermitian(A), uplo, Val{false}) end ### Default to no pivoting (and storing of upper factor) when not explicit -cholfact(A::Hermitian) = cholfact(A, Val{false}) -cholfact{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}) = cholfact(A, Val{false}) +cholfact(A::Hermitian, uplo::Symbol = :U) = cholfact(A, uplo, Val{false}) +cholfact{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}, uplo::Symbol = :U) = + cholfact(A, uplo, Val{false}) #### for StridedMatrices, check that matrix is symmetric/Hermitian function cholfact(A::StridedMatrix, uplo::Symbol = :U) ishermitian(A) || non_hermitian_error("cholfact") - return cholfact(Hermitian(A, uplo)) + return cholfact(Hermitian(A), uplo) end ## With pivoting -cholfact(A::Hermitian, ::Type{Val{true}}; tol = 0.0) = +cholfact(A::Hermitian, uplo::Symbol, ::Type{Val{true}}; tol = 0.0) = cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), - Val{true}; tol = tol) -cholfact{T<:Real,S<:StridedMatrix}(A::RealHermSymComplexHerm{T,S}, ::Type{Val{true}}; tol = 0.0) = + uplo, Val{true}; tol = tol) +cholfact{T<:Real,S<:StridedMatrix}(A::RealHermSymComplexHerm{T,S}, uplo::Symbol, + ::Type{Val{true}}; tol = 0.0) = cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), - Val{true}; tol = tol) + uplo, Val{true}; tol = tol) ### for StridedMatrices, check that matrix is symmetric/Hermitian """ - cholfact(A, [uplo::Symbol,] Val{true}; tol = 0.0) -> CholeskyPivoted + cholfact(A, uplo::Symbol, Val{true}; tol = 0.0) -> CholeskyPivoted Compute the pivoted Cholesky factorization of a dense symmetric positive semi-definite matrix `A` -and return a `CholeskyPivoted` factorization. The matrix `A` can either be a `Symmetric` or `Hermitian` -`StridedMatrix` or a *perfectly* symmetric or Hermitian `StridedMatrix`. In the latter case, -the optional argument `uplo` may be `:L` for using the lower part or `:U` for the upper part of `A`. +and return a `CholeskyPivoted` factorization. +The `uplo` argument may be `:L` for using the lower part or `:U` for the upper part of `A`. The default is to use `:U`. The triangular Cholesky factor can be obtained from the factorization `F` with: `F[:L]` and `F[:U]`. The following functions are available for `PivotedCholesky` objects: `size`, `\\`, `inv`, `det`, and `rank`. @@ -307,7 +300,7 @@ For negative values, the tolerance is the machine precision. """ function cholfact(A::StridedMatrix, uplo::Symbol, ::Type{Val{true}}; tol = 0.0) ishermitian(A) || non_hermitian_error("cholfact") - return cholfact(Hermitian(A, uplo), Val{true}; tol = tol) + return cholfact(Hermitian(A), uplo, Val{true}; tol = tol) end ## Number diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index ff1a4808cd8d5..0a06293817286 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -308,17 +308,17 @@ Linear algebra functions in Julia are largely implemented by calling functions f Compute the square root of a non-negative number ``x``\ . -.. function:: cholfact(A, [uplo::Symbol,] Val{false}) -> Cholesky +.. function:: cholfact(A, uplo::Symbol, Val{false}) -> Cholesky .. Docstring generated from Julia source - Compute the Cholesky factorization of a dense symmetric positive definite matrix ``A`` and return a ``Cholesky`` factorization. The matrix ``A`` can either be a ``Symmetric`` or ``Hermitian`` ``StridedMatrix`` or a *perfectly* symmetric or Hermitian ``StridedMatrix``\ . In the latter case, the optional argument ``uplo`` may be ``:L`` for using the lower part or ``:U`` for the upper part of ``A``\ . The default is to use ``:U``\ . The triangular Cholesky factor can be obtained from the factorization ``F`` with: ``F[:L]`` and ``F[:U]``\ . The following functions are available for ``Cholesky`` objects: ``size``\ , ``\``\ , ``inv``\ , ``det``\ . A ``PosDefException`` exception is thrown in case the matrix is not positive definite. + Compute the Cholesky factorization of a dense symmetric positive definite matrix ``A`` and return a ``Cholesky`` factorization. The ``uplo`` argument may be ``:L`` for using the lower part or ``:U`` for the upper part of ``A``\ . The default is to use ``:U``\ . The triangular Cholesky factor can be obtained from the factorization ``F`` with: ``F[:L]`` and ``F[:U]``\ . The following functions are available for ``Cholesky`` objects: ``size``\ , ``\``\ , ``inv``\ , ``det``\ . A ``PosDefException`` exception is thrown in case the matrix is not positive definite. -.. function:: cholfact(A, [uplo::Symbol,] Val{true}; tol = 0.0) -> CholeskyPivoted +.. function:: cholfact(A, uplo::Symbol, Val{true}; tol = 0.0) -> CholeskyPivoted .. Docstring generated from Julia source - Compute the pivoted Cholesky factorization of a dense symmetric positive semi-definite matrix ``A`` and return a ``CholeskyPivoted`` factorization. The matrix ``A`` can either be a ``Symmetric`` or ``Hermitian`` ``StridedMatrix`` or a *perfectly* symmetric or Hermitian ``StridedMatrix``\ . In the latter case, the optional argument ``uplo`` may be ``:L`` for using the lower part or ``:U`` for the upper part of ``A``\ . The default is to use ``:U``\ . The triangular Cholesky factor can be obtained from the factorization ``F`` with: ``F[:L]`` and ``F[:U]``\ . The following functions are available for ``PivotedCholesky`` objects: ``size``\ , ``\``\ , ``inv``\ , ``det``\ , and ``rank``\ . The argument ``tol`` determines the tolerance for determining the rank. For negative values, the tolerance is the machine precision. + Compute the pivoted Cholesky factorization of a dense symmetric positive semi-definite matrix ``A`` and return a ``CholeskyPivoted`` factorization. The ``uplo`` argument may be ``:L`` for using the lower part or ``:U`` for the upper part of ``A``\ . The default is to use ``:U``\ . The triangular Cholesky factor can be obtained from the factorization ``F`` with: ``F[:L]`` and ``F[:U]``\ . The following functions are available for ``PivotedCholesky`` objects: ``size``\ , ``\``\ , ``inv``\ , ``det``\ , and ``rank``\ . The argument ``tol`` determines the tolerance for determining the rank. For negative values, the tolerance is the machine precision. .. function:: cholfact(A; shift = 0.0, perm = Int[]) -> CHOLMOD.Factor @@ -344,13 +344,13 @@ Linear algebra functions in Julia are largely implemented by calling functions f This method uses the CHOLMOD library from SuiteSparse, which only supports doubles or complex doubles. Input matrices not of those element types will be converted to ``SparseMatrixCSC{Float64}`` or ``SparseMatrixCSC{Complex128}`` as appropriate. -.. function:: cholfact!(A, [uplo::Symbol,] Val{false}) -> Cholesky +.. function:: cholfact!(A, uplo::Symbol, Val{false}) -> Cholesky .. Docstring generated from Julia source The same as ``cholfact``\ , but saves space by overwriting the input ``A``\ , instead of creating a copy. An ``InexactError`` exception is thrown if the factorisation produces a number not representable by the element type of ``A``\ , e.g. for integer types. -.. function:: cholfact!(A, [uplo::Symbol,] Val{true}; tol = 0.0) -> CholeskyPivoted +.. function:: cholfact!(A, uplo::Symbol, Val{true}; tol = 0.0) -> CholeskyPivoted .. Docstring generated from Julia source diff --git a/test/linalg/cholesky.jl b/test/linalg/cholesky.jl index bb0657c799838..00e4696b4f12b 100644 --- a/test/linalg/cholesky.jl +++ b/test/linalg/cholesky.jl @@ -74,15 +74,15 @@ for eltya in (Float32, Float64, Complex64, Complex128, BigFloat, Int) @test full(lapd) ≈ apd l = lapd[:L] @test l*l' ≈ apd - @test capd[:U] ≈ lapd[:U] - @test lapd[:L] ≈ capd[:L] + @test triu(capd.factors) ≈ lapd[:U] + @test tril(lapd.factors) ≈ capd[:L] if eltya <: Real capds = cholfact(apds) - lapds = cholfact(full(apds), :L) + lapds = cholfact(apds, :L) ls = lapds[:L] @test ls*ls' ≈ apd - @test capds[:U] ≈ lapds[:U] - @test lapds[:L] ≈ capds[:L] + @test triu(capds.factors) ≈ lapds[:U] + @test tril(lapds.factors) ≈ capds[:L] end #pivoted upper Cholesky From dccf0cca11d7dbdc1dbb365db6bce4acc7f71ed9 Mon Sep 17 00:00:00 2001 From: Andreas Noack Date: Thu, 11 Aug 2016 21:31:58 -0400 Subject: [PATCH 79/80] Remove redundant uplo argument in chol family. When using Hermitian and Symmetric, it is redundant to also have an uplo argument and occasionally, it also gave the wrong result. This can therefore be considered a bugfix. (cherry picked from commit 608510306b3907e8ad0d7171b1a530169d81d6e2) ref #17985 --- base/deprecated.jl | 3 ++ base/linalg/cholesky.jl | 97 ++++++++++++++++++++++------------------- doc/stdlib/linalg.rst | 12 ++--- test/linalg/cholesky.jl | 2 +- 4 files changed, 62 insertions(+), 52 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index ac4eb4e1c5347..002da08644150 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -775,6 +775,9 @@ function transpose(x) return x end +@deprecate cholfact!(A::Base.LinAlg.HermOrSym, uplo::Symbol, ::Type{Val{false}}) cholfact!(A, Val{false}) +@deprecate cholfact!(A::Base.LinAlg.HermOrSym, uplo::Symbol = :U) cholfact!(A) + # During the 0.5 development cycle, do not add any deprecations below this line # To be deprecated in 0.6 diff --git a/base/linalg/cholesky.jl b/base/linalg/cholesky.jl index e961ef09f5a57..246f1b3c8c140 100644 --- a/base/linalg/cholesky.jl +++ b/base/linalg/cholesky.jl @@ -121,8 +121,10 @@ non_hermitian_error(f) = throw(ArgumentError("matrix is not symmetric/" * # chol!. Destructive methods for computing Cholesky factor of real symmetric or Hermitian # matrix -chol!(A::Hermitian) = _chol!(A.data, UpperTriangular) -chol!{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}) = _chol!(A.data, UpperTriangular) +chol!(A::Hermitian) = + _chol!(A.uplo == 'U' ? A.data : LinAlg.copytri!(A.data, 'L', true), UpperTriangular) +chol!{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}) = + _chol!(A.uplo == 'U' ? A.data : LinAlg.copytri!(A.data, 'L', true), UpperTriangular) function chol!(A::StridedMatrix) ishermitian(A) || non_hermitian_error("chol!") return _chol!(A, UpperTriangular) @@ -135,14 +137,22 @@ end function chol(A::Hermitian) T = promote_type(typeof(chol(one(eltype(A)))), Float32) AA = similar(A, T, size(A)) - copy!(AA, A.data) - chol!(Hermitian(AA)) + if A.uplo == 'U' + copy!(AA, A.data) + else + Base.ccopy!(AA, A.data) + end + chol!(Hermitian(AA, :U)) end function chol{T<:Real,S<:AbstractMatrix}(A::Symmetric{T,S}) TT = promote_type(typeof(chol(one(T))), Float32) AA = similar(A, TT, size(A)) - copy!(AA, A.data) - chol!(Hermitian(AA)) + if A.uplo == 'U' + copy!(AA, A.data) + else + Base.ccopy!(AA, A.data) + end + chol!(Hermitian(AA, :U)) end ## for StridedMatrices, check that matrix is symmetric/Hermitian @@ -170,15 +180,15 @@ chol(x::Number, args...) = _chol!(x, nothing) # cholfact!. Destructive methods for computing Cholesky factorization of real symmetric # or Hermitian matrix ## No pivoting -function cholfact!(A::Hermitian, uplo::Symbol, ::Type{Val{false}}) - if uplo == :U +function cholfact!(A::Hermitian, ::Type{Val{false}}) + if A.uplo == 'U' Cholesky(_chol!(A.data, UpperTriangular).data, 'U') else Cholesky(_chol!(A.data, LowerTriangular).data, 'L') end end -function cholfact!{T<:Real,S}(A::Symmetric{T,S}, uplo::Symbol, ::Type{Val{false}}) - if uplo == :U +function cholfact!{T<:Real,S}(A::Symmetric{T,S}, ::Type{Val{false}}) + if A.uplo == 'U' Cholesky(_chol!(A.data, UpperTriangular).data, 'U') else Cholesky(_chol!(A.data, LowerTriangular).data, 'L') @@ -187,7 +197,7 @@ end ### for StridedMatrices, check that matrix is symmetric/Hermitian """ - cholfact!(A, uplo::Symbol, Val{false}) -> Cholesky + cholfact!(A, [uplo::Symbol,] Val{false}) -> Cholesky The same as `cholfact`, but saves space by overwriting the input `A`, instead of creating a copy. An `InexactError` exception is thrown if the factorisation @@ -196,37 +206,36 @@ integer types. """ function cholfact!(A::StridedMatrix, uplo::Symbol, ::Type{Val{false}}) ishermitian(A) || non_hermitian_error("cholfact!") - return cholfact!(Hermitian(A), uplo, Val{false}) + return cholfact!(Hermitian(A, uplo), Val{false}) end ### Default to no pivoting (and storing of upper factor) when not explicit -cholfact!(A::Hermitian, uplo::Symbol = :U) = cholfact!(A, uplo, Val{false}) -cholfact!{T<:Real,S}(A::Symmetric{T,S}, uplo::Symbol = :U) = cholfact!(A, uplo, Val{false}) +cholfact!(A::Hermitian) = cholfact!(A, Val{false}) +cholfact!{T<:Real,S}(A::Symmetric{T,S}) = cholfact!(A, Val{false}) #### for StridedMatrices, check that matrix is symmetric/Hermitian function cholfact!(A::StridedMatrix, uplo::Symbol = :U) ishermitian(A) || non_hermitian_error("cholfact!") - return cholfact!(Hermitian(A), uplo) + return cholfact!(Hermitian(A, uplo)) end ## With pivoting ### BLAS/LAPACK element types function cholfact!{T<:BlasReal,S<:StridedMatrix}(A::RealHermSymComplexHerm{T,S}, - uplo::Symbol, ::Type{Val{true}}; tol = 0.0) - uplochar = char_uplo(uplo) - AA, piv, rank, info = LAPACK.pstrf!(uplochar, A.data, tol) - return CholeskyPivoted{eltype(AA),typeof(AA)}(AA, uplochar, piv, rank, tol, info) + ::Type{Val{true}}; tol = 0.0) + AA, piv, rank, info = LAPACK.pstrf!(A.uplo, A.data, tol) + return CholeskyPivoted{eltype(AA),typeof(AA)}(AA, A.uplo, piv, rank, tol, info) end ### Non BLAS/LAPACK element types (generic). Since generic fallback for pivoted Cholesky ### is not implemented yet we throw an error -cholfact!{T<:Real,S}(A::RealHermSymComplexHerm{T,S}, uplo::Symbol, ::Type{Val{true}}; +cholfact!{T<:Real,S}(A::RealHermSymComplexHerm{T,S}, ::Type{Val{true}}; tol = 0.0) = throw(ArgumentError("generic pivoted Cholesky factorization is not implemented yet")) ### for StridedMatrices, check that matrix is symmetric/Hermitian """ - cholfact!(A, uplo::Symbol, Val{true}; tol = 0.0) -> CholeskyPivoted + cholfact!(A, [uplo::Symbol,] Val{true}; tol = 0.0) -> CholeskyPivoted The same as `cholfact`, but saves space by overwriting the input `A`, instead of creating a copy. An `InexactError` exception is thrown if the @@ -235,26 +244,25 @@ e.g. for integer types. """ function cholfact!(A::StridedMatrix, uplo::Symbol, ::Type{Val{true}}; tol = 0.0) ishermitian(A) || non_hermitian_error("cholfact!") - return cholfact!(Hermitian(A), uplo, Val{true}; tol = tol) + return cholfact!(Hermitian(A, uplo), Val{true}; tol = tol) end - - # cholfact. Non-destructive methods for computing Cholesky factorization of real symmetric # or Hermitian matrix ## No pivoting -cholfact(A::Hermitian, uplo::Symbol, ::Type{Val{false}}) = - cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), uplo, Val{false}) -cholfact{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}, uplo::Symbol, ::Type{Val{false}}) = - cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), uplo, Val{false}) +cholfact(A::Hermitian, ::Type{Val{false}}) = + cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), Val{false}) +cholfact{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}, ::Type{Val{false}}) = + cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), Val{false}) ### for StridedMatrices, check that matrix is symmetric/Hermitian """ - cholfact(A, uplo::Symbol, Val{false}) -> Cholesky + cholfact(A, [uplo::Symbol,] Val{false}) -> Cholesky Compute the Cholesky factorization of a dense symmetric positive definite matrix `A` -and return a `Cholesky` factorization. -The `uplo` argument may be `:L` for using the lower part or `:U` for the upper part of `A`. +and return a `Cholesky` factorization. The matrix `A` can either be a `Symmetric` or `Hermitian` +`StridedMatrix` or a *perfectly* symmetric or Hermitian `StridedMatrix`. In the latter case, +the optional argument `uplo` may be `:L` for using the lower part or `:U` for the upper part of `A`. The default is to use `:U`. The triangular Cholesky factor can be obtained from the factorization `F` with: `F[:L]` and `F[:U]`. The following functions are available for `Cholesky` objects: `size`, `\\`, `inv`, `det`. @@ -262,36 +270,35 @@ A `PosDefException` exception is thrown in case the matrix is not positive defin """ function cholfact(A::StridedMatrix, uplo::Symbol, ::Type{Val{false}}) ishermitian(A) || non_hermitian_error("cholfact") - return cholfact(Hermitian(A), uplo, Val{false}) + return cholfact(Hermitian(A, uplo), Val{false}) end ### Default to no pivoting (and storing of upper factor) when not explicit -cholfact(A::Hermitian, uplo::Symbol = :U) = cholfact(A, uplo, Val{false}) -cholfact{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}, uplo::Symbol = :U) = - cholfact(A, uplo, Val{false}) +cholfact(A::Hermitian) = cholfact(A, Val{false}) +cholfact{T<:Real,S<:StridedMatrix}(A::Symmetric{T,S}) = cholfact(A, Val{false}) #### for StridedMatrices, check that matrix is symmetric/Hermitian function cholfact(A::StridedMatrix, uplo::Symbol = :U) ishermitian(A) || non_hermitian_error("cholfact") - return cholfact(Hermitian(A), uplo) + return cholfact(Hermitian(A, uplo)) end ## With pivoting -cholfact(A::Hermitian, uplo::Symbol, ::Type{Val{true}}; tol = 0.0) = +cholfact(A::Hermitian, ::Type{Val{true}}; tol = 0.0) = cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), - uplo, Val{true}; tol = tol) -cholfact{T<:Real,S<:StridedMatrix}(A::RealHermSymComplexHerm{T,S}, uplo::Symbol, - ::Type{Val{true}}; tol = 0.0) = + Val{true}; tol = tol) +cholfact{T<:Real,S<:StridedMatrix}(A::RealHermSymComplexHerm{T,S}, ::Type{Val{true}}; tol = 0.0) = cholfact!(copy_oftype(A, promote_type(typeof(chol(one(eltype(A)))),Float32)), - uplo, Val{true}; tol = tol) + Val{true}; tol = tol) ### for StridedMatrices, check that matrix is symmetric/Hermitian """ - cholfact(A, uplo::Symbol, Val{true}; tol = 0.0) -> CholeskyPivoted + cholfact(A, [uplo::Symbol,] Val{true}; tol = 0.0) -> CholeskyPivoted Compute the pivoted Cholesky factorization of a dense symmetric positive semi-definite matrix `A` -and return a `CholeskyPivoted` factorization. -The `uplo` argument may be `:L` for using the lower part or `:U` for the upper part of `A`. +and return a `CholeskyPivoted` factorization. The matrix `A` can either be a `Symmetric` or `Hermitian` +`StridedMatrix` or a *perfectly* symmetric or Hermitian `StridedMatrix`. In the latter case, +the optional argument `uplo` may be `:L` for using the lower part or `:U` for the upper part of `A`. The default is to use `:U`. The triangular Cholesky factor can be obtained from the factorization `F` with: `F[:L]` and `F[:U]`. The following functions are available for `PivotedCholesky` objects: `size`, `\\`, `inv`, `det`, and `rank`. @@ -300,7 +307,7 @@ For negative values, the tolerance is the machine precision. """ function cholfact(A::StridedMatrix, uplo::Symbol, ::Type{Val{true}}; tol = 0.0) ishermitian(A) || non_hermitian_error("cholfact") - return cholfact(Hermitian(A), uplo, Val{true}; tol = tol) + return cholfact(Hermitian(A, uplo), Val{true}; tol = tol) end ## Number diff --git a/doc/stdlib/linalg.rst b/doc/stdlib/linalg.rst index 0a06293817286..ff1a4808cd8d5 100644 --- a/doc/stdlib/linalg.rst +++ b/doc/stdlib/linalg.rst @@ -308,17 +308,17 @@ Linear algebra functions in Julia are largely implemented by calling functions f Compute the square root of a non-negative number ``x``\ . -.. function:: cholfact(A, uplo::Symbol, Val{false}) -> Cholesky +.. function:: cholfact(A, [uplo::Symbol,] Val{false}) -> Cholesky .. Docstring generated from Julia source - Compute the Cholesky factorization of a dense symmetric positive definite matrix ``A`` and return a ``Cholesky`` factorization. The ``uplo`` argument may be ``:L`` for using the lower part or ``:U`` for the upper part of ``A``\ . The default is to use ``:U``\ . The triangular Cholesky factor can be obtained from the factorization ``F`` with: ``F[:L]`` and ``F[:U]``\ . The following functions are available for ``Cholesky`` objects: ``size``\ , ``\``\ , ``inv``\ , ``det``\ . A ``PosDefException`` exception is thrown in case the matrix is not positive definite. + Compute the Cholesky factorization of a dense symmetric positive definite matrix ``A`` and return a ``Cholesky`` factorization. The matrix ``A`` can either be a ``Symmetric`` or ``Hermitian`` ``StridedMatrix`` or a *perfectly* symmetric or Hermitian ``StridedMatrix``\ . In the latter case, the optional argument ``uplo`` may be ``:L`` for using the lower part or ``:U`` for the upper part of ``A``\ . The default is to use ``:U``\ . The triangular Cholesky factor can be obtained from the factorization ``F`` with: ``F[:L]`` and ``F[:U]``\ . The following functions are available for ``Cholesky`` objects: ``size``\ , ``\``\ , ``inv``\ , ``det``\ . A ``PosDefException`` exception is thrown in case the matrix is not positive definite. -.. function:: cholfact(A, uplo::Symbol, Val{true}; tol = 0.0) -> CholeskyPivoted +.. function:: cholfact(A, [uplo::Symbol,] Val{true}; tol = 0.0) -> CholeskyPivoted .. Docstring generated from Julia source - Compute the pivoted Cholesky factorization of a dense symmetric positive semi-definite matrix ``A`` and return a ``CholeskyPivoted`` factorization. The ``uplo`` argument may be ``:L`` for using the lower part or ``:U`` for the upper part of ``A``\ . The default is to use ``:U``\ . The triangular Cholesky factor can be obtained from the factorization ``F`` with: ``F[:L]`` and ``F[:U]``\ . The following functions are available for ``PivotedCholesky`` objects: ``size``\ , ``\``\ , ``inv``\ , ``det``\ , and ``rank``\ . The argument ``tol`` determines the tolerance for determining the rank. For negative values, the tolerance is the machine precision. + Compute the pivoted Cholesky factorization of a dense symmetric positive semi-definite matrix ``A`` and return a ``CholeskyPivoted`` factorization. The matrix ``A`` can either be a ``Symmetric`` or ``Hermitian`` ``StridedMatrix`` or a *perfectly* symmetric or Hermitian ``StridedMatrix``\ . In the latter case, the optional argument ``uplo`` may be ``:L`` for using the lower part or ``:U`` for the upper part of ``A``\ . The default is to use ``:U``\ . The triangular Cholesky factor can be obtained from the factorization ``F`` with: ``F[:L]`` and ``F[:U]``\ . The following functions are available for ``PivotedCholesky`` objects: ``size``\ , ``\``\ , ``inv``\ , ``det``\ , and ``rank``\ . The argument ``tol`` determines the tolerance for determining the rank. For negative values, the tolerance is the machine precision. .. function:: cholfact(A; shift = 0.0, perm = Int[]) -> CHOLMOD.Factor @@ -344,13 +344,13 @@ Linear algebra functions in Julia are largely implemented by calling functions f This method uses the CHOLMOD library from SuiteSparse, which only supports doubles or complex doubles. Input matrices not of those element types will be converted to ``SparseMatrixCSC{Float64}`` or ``SparseMatrixCSC{Complex128}`` as appropriate. -.. function:: cholfact!(A, uplo::Symbol, Val{false}) -> Cholesky +.. function:: cholfact!(A, [uplo::Symbol,] Val{false}) -> Cholesky .. Docstring generated from Julia source The same as ``cholfact``\ , but saves space by overwriting the input ``A``\ , instead of creating a copy. An ``InexactError`` exception is thrown if the factorisation produces a number not representable by the element type of ``A``\ , e.g. for integer types. -.. function:: cholfact!(A, uplo::Symbol, Val{true}; tol = 0.0) -> CholeskyPivoted +.. function:: cholfact!(A, [uplo::Symbol,] Val{true}; tol = 0.0) -> CholeskyPivoted .. Docstring generated from Julia source diff --git a/test/linalg/cholesky.jl b/test/linalg/cholesky.jl index 00e4696b4f12b..1c61f12060e5f 100644 --- a/test/linalg/cholesky.jl +++ b/test/linalg/cholesky.jl @@ -78,7 +78,7 @@ for eltya in (Float32, Float64, Complex64, Complex128, BigFloat, Int) @test tril(lapd.factors) ≈ capd[:L] if eltya <: Real capds = cholfact(apds) - lapds = cholfact(apds, :L) + lapds = cholfact(Symmetric(apds.data, :L)) ls = lapds[:L] @test ls*ls' ≈ apd @test triu(capds.factors) ≈ lapds[:U] From 688c3d6639f86bfcb2626317b7e547c21a021e3e Mon Sep 17 00:00:00 2001 From: Katharine Hyatt Date: Fri, 12 Aug 2016 00:40:50 -0700 Subject: [PATCH 80/80] Fix atahn to atanh (#17977) * Fix atahn to atanh * More tests (cherry picked from commit 5838a12ba037f40b31affe5b408edcf9ce4bed28) --- base/special/trig.jl | 2 +- test/math.jl | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/base/special/trig.jl b/base/special/trig.jl index 2ad94211c590f..4a4e0b94170a7 100644 --- a/base/special/trig.jl +++ b/base/special/trig.jl @@ -321,7 +321,7 @@ end for (tfa, tfainv, hfa, hfainv, fn) in ((:asec, :acos, :asech, :acosh, "secant"), (:acsc, :asin, :acsch, :asinh, "cosecant"), - (:acot, :atan, :acoth, :atahn, "cotangent")) + (:acot, :atan, :acoth, :atanh, "cotangent")) tname = string(tfa) hname = string(hfa) @eval begin diff --git a/test/math.jl b/test/math.jl index 2394bd460e4ea..26a744b3211f0 100644 --- a/test/math.jl +++ b/test/math.jl @@ -192,6 +192,16 @@ end @test exp2(Float16(2.)) ≈ exp2(2.) @test log(e) == 1 +#test abstractarray trig fxns +TAA = rand(2,2) +TAA = (TAA + TAA.')/2. +STAA = Symmetric(TAA) +@test full(atanh(STAA)) == atanh(TAA) +@test full(asinh(STAA)) == asinh(TAA) +@test full(acosh(STAA+Symmetric(ones(TAA)))) == acosh(TAA+ones(TAA)) +@test full(acsch(STAA+Symmetric(ones(TAA)))) == acsch(TAA+ones(TAA)) +@test full(acoth(STAA+Symmetric(ones(TAA)))) == acoth(TAA+ones(TAA)) + # check exp2(::Integer) matches exp2(::Float) for ii in -2048:2048 expected = exp2(float(ii))