Merge pull request #23931 from JuliaLang/jb/move_mmap_sharedarrays

move Mmap and SharedArrays to stdlib
JuliaLang · Sep 30, 2017 · f47acd5 · f47acd5 · nanosoldier · Sep 30, 2017
2 parents a6022fd + af97d70
commit f47acd5
Show file tree

Hide file tree

Showing 21 changed files with 387 additions and 353 deletions.
diff --git a/Make.inc b/Make.inc
@@ -1123,19 +1123,19 @@ endef
 endif
 
 define symlink_target
-CLEAN_TARGETS += clean-$(2)/$(1)
-clean-$$(abspath $(2)/$(1)):
+CLEAN_TARGETS += clean-$(2)/$(3)
+clean-$$(abspath $(2)/$(3)):
 ifeq ($(BUILD_OS), WINNT)
-	@-cmd //C rmdir $$(call mingw_to_dos,$(2)/$(1),cd $(2) &&)
+	@-cmd //C rmdir $$(call mingw_to_dos,$(2)/$(3),cd $(2) &&)
 else
-	@-rm $$(abspath $(2)/$(1))
+	@-rm $$(abspath $(2)/$(3))
 endif
-$$(subst $$(abspath $(JULIAHOME))/,,$$(abspath $(2)/$(1))): $$(abspath $(2)/$(1))
-$$(abspath $(2)/$(1)): | $$(abspath $(2))
+$$(subst $$(abspath $(JULIAHOME))/,,$$(abspath $(2)/$(3))): $$(abspath $(2)/$(3))
+$$(abspath $(2)/$(3)): | $$(abspath $(2))
 ifeq ($(BUILD_OS), WINNT)
-	@cmd //C mklink //J $$(call mingw_to_dos,$(2)/$(1),cd $(2) &&) $$(call mingw_to_dos,$(1),)
+	@cmd //C mklink //J $$(call mingw_to_dos,$(2)/$(3),cd $(2) &&) $$(call mingw_to_dos,$(1),)
 else ifneq (,$(findstring CYGWIN,$(BUILD_OS)))
-	@cmd /C mklink /J $$(call cygpath_w,$(2)/$(1)) $$(call cygpath_w,$(1))
+	@cmd /C mklink /J $$(call cygpath_w,$(2)/$(3)) $$(call cygpath_w,$(1))
 else ifdef JULIA_VAGRANT_BUILD
 	@cp -R $$(abspath $(1)) $$@
 else

diff --git a/Makefile b/Makefile
@@ -49,12 +49,10 @@ configure:
 endif
 
 $(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir))))
-$(foreach link,base test,$(eval $(call symlink_target,$(link),$(build_datarootdir)/julia)))
-$(eval $(call symlink_target,stdlib,$(build_datarootdir)/julia/site))
+$(foreach link,base test,$(eval $(call symlink_target,$(link),$(build_datarootdir)/julia,$(link))))
 
 build_defaultpkgdir = $(build_datarootdir)/julia/site/$(shell echo $(VERSDIR))
-$(build_defaultpkgdir): $(build_datarootdir)/julia/site/stdlib
-	@mv $(build_datarootdir)/julia/site/stdlib $@
+$(eval $(call symlink_target,stdlib,$(build_datarootdir)/julia/site,$(shell echo $(VERSDIR))))
 
 julia_flisp.boot.inc.phony: julia-deps
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/src julia_flisp.boot.inc.phony

diff --git a/base/deprecated.jl b/base/deprecated.jl
@@ -1014,14 +1014,6 @@ isempty(::Task) = error("isempty not defined for Tasks")
 @deprecate Array(::Type{T}, m::Integer,n::Integer) where {T}            Array{T}(Int(m),Int(n))
 @deprecate Array(::Type{T}, m::Integer,n::Integer,o::Integer) where {T} Array{T}(Int(m),Int(n),Int(o))
 
-# Likewise for SharedArrays
-@deprecate SharedArray(::Type{T}, dims::Dims{N}; kwargs...) where {T,N} SharedArray{T}(dims; kwargs...)
-@deprecate SharedArray(::Type{T}, dims::Int...; kwargs...) where {T}    SharedArray{T}(dims...; kwargs...)
-@deprecate(SharedArray(filename::AbstractString, ::Type{T}, dims::NTuple{N,Int}, offset; kwargs...) where {T,N},
-           SharedArray{T}(filename, dims, offset; kwargs...))
-@deprecate(SharedArray(filename::AbstractString, ::Type{T}, dims::NTuple, offset; kwargs...) where {T},
-           SharedArray{T}(filename, dims, offset; kwargs...))
-
 @noinline function is_intrinsic_expr(@nospecialize(x))
     Base.depwarn("is_intrinsic_expr is deprecated. There are no intrinsic functions anymore.", :is_intrinsic_expr)
     return false
@@ -1381,6 +1373,10 @@ end
 export Test
 deprecate(@__MODULE__, :Test)
 
+@deprecate_moved SharedArray "SharedArrays" true true
+
+@deprecate_binding Mmap nothing true ", run `using Mmap` instead"
+
 # PR #21709
 @deprecate cov(x::AbstractVector, corrected::Bool) cov(x, corrected=corrected)
 @deprecate cov(x::AbstractMatrix, vardim::Int, corrected::Bool) cov(x, vardim, corrected=corrected)

diff --git a/base/exports.jl b/base/exports.jl
@@ -11,7 +11,6 @@ export
     Sys,
     Libc,
     Libdl,
-    Mmap,
     LinAlg,
     BLAS,
     LAPACK,
@@ -104,9 +103,6 @@ export
     AbstractSerializer,
     SerializationState,
     Set,
-    SharedArray,
-    SharedMatrix,
-    SharedVector,
     StepRange,
     StepRangeLen,
     StridedArray,
@@ -1088,11 +1084,6 @@ export
     HTML,
     Text,
 
-# shared arrays
-    sdata,
-    indexpids,
-    localindexes,
-
 # paths and file names
     abspath,
     basename,

diff --git a/base/sysimg.jl b/base/sysimg.jl
@@ -355,10 +355,6 @@ using .Serializer
 import .Serializer: serialize, deserialize
 include("channels.jl")
 
-# memory-mapped and shared arrays
-include("mmap.jl")
-import .Mmap
-
 # utilities - timing, help, edit
 include("deepcopy.jl")
 include("interactiveutil.jl")
@@ -413,7 +409,6 @@ include("asyncmap.jl")
 
 include("distributed/Distributed.jl")
 using .Distributed
-include("sharedarray.jl")
 
 # code loading
 include("loading.jl")

diff --git a/doc/make.jl b/doc/make.jl
@@ -16,13 +16,20 @@ end
 
 # Documenter Setup.
 
+symlink_q(tgt, link) = isfile(link) || symlink(tgt, link)
+cp_q(src, dest) = isfile(dest) || cp(src, dest)
+
 # make links for stdlib package docs
 if Sys.iswindows()
-    cp("../stdlib/DelimitedFiles/docs/src/index.md", "src/stdlib/delimitedfiles.md")
-    cp("../stdlib/Test/docs/src/index.md", "src/stdlib/test.md")
+    cp_q("../stdlib/DelimitedFiles/docs/src/index.md", "src/stdlib/delimitedfiles.md")
+    cp_q("../stdlib/Test/docs/src/index.md", "src/stdlib/test.md")
+    cp_q("../stdlib/Mmap/docs/src/index.md", "src/stdlib/mmap.md")
+    cp_q("../stdlib/SharedArrays/docs/src/index.md", "src/stdlib/sharedarrays.md")
 else
-    symlink("../../../stdlib/DelimitedFiles/docs/src/index.md", "src/stdlib/delimitedfiles.md")
-    symlink("../../../stdlib/Test/docs/src/index.md", "src/stdlib/test.md")
+    symlink_q("../../../stdlib/DelimitedFiles/docs/src/index.md", "src/stdlib/delimitedfiles.md")
+    symlink_q("../../../stdlib/Test/docs/src/index.md", "src/stdlib/test.md")
+    symlink_q("../../../stdlib/Mmap/docs/src/index.md", "src/stdlib/mmap.md")
+    symlink_q("../../../stdlib/SharedArrays/docs/src/index.md", "src/stdlib/sharedarrays.md")
 end
 
 const PAGES = [
@@ -126,11 +133,11 @@ const PAGES = [
     ],
 ]
 
-using DelimitedFiles, Test
+using DelimitedFiles, Test, Mmap, SharedArrays
 
 makedocs(
     build     = joinpath(pwd(), "_build/html/en"),
-    modules   = [Base, Core, BuildSysImg, DelimitedFiles, Test],
+    modules   = [Base, Core, BuildSysImg, DelimitedFiles, Test, Mmap, SharedArrays],
     clean     = false,
     doctest   = "doctest" in ARGS,
     linkcheck = "linkcheck" in ARGS,

diff --git a/doc/src/manual/parallel-computing.md b/doc/src/manual/parallel-computing.md
@@ -858,7 +858,9 @@ julia> addprocs(3)
  3
  4
 
-julia> S = SharedArray{Int,2}((3,4), init = S -> S[Base.localindexes(S)] = myid())
+julia> @everywhere using SharedArrays
+
+julia> S = SharedArray{Int,2}((3,4), init = S -> S[localindexes(S)] = myid())
 3×4 SharedArray{Int64,2}:
  2  2  3  4
  2  3  3  4
@@ -874,7 +876,7 @@ julia> S
  2  7  4  4
 ```
 
-[`Base.localindexes`](@ref) provides disjoint one-dimensional ranges of indexes, and is sometimes
+[`SharedArrays.localindexes`](@ref) provides disjoint one-dimensional ranges of indexes, and is sometimes
 convenient for splitting up tasks among processes. You can, of course, divide the work any way
 you wish:
 

diff --git a/doc/src/stdlib/.gitignore b/doc/src/stdlib/.gitignore
@@ -1,2 +1,4 @@
 delimitedfiles.md
 test.md
+mmap.md
+sharedarrays.md
diff --git a/doc/src/stdlib/index.md b/doc/src/stdlib/index.md
@@ -24,3 +24,5 @@
   * [Dynamic Linker](@ref)
   * [Profiling](@ref lib-profiling)
   * [SIMD Support](@ref)
+  * [Memory-mapped I/O](@ref)
+  * [Shared Arrays](@ref)
diff --git a/doc/src/stdlib/io-network.md b/doc/src/stdlib/io-network.md
@@ -136,14 +136,6 @@ Base.Multimedia.TextDisplay
 Base.Multimedia.istextmime
 ```
 
-## Memory-mapped I/O
-
-```@docs
-Base.Mmap.Anonymous
-Base.Mmap.mmap
-Base.Mmap.sync!
-```
-
 ## Network I/O
 
 ```@docs

diff --git a/doc/src/stdlib/parallel.md b/doc/src/stdlib/parallel.md
@@ -83,16 +83,6 @@ Base.cluster_cookie()
 Base.cluster_cookie(::Any)
 ```
 
-## Shared Arrays
-
-```@docs
-Base.SharedArray
-Base.procs(::SharedArray)
-Base.sdata
-Base.indexpids
-Base.localindexes
-```
-
 ## Multi-Threading
 
 This experimental interface supports Julia's multi-threading capabilities. Types and functions

diff --git a/stdlib/DelimitedFiles/src/DelimitedFiles.jl b/stdlib/DelimitedFiles/src/DelimitedFiles.jl
@@ -4,6 +4,8 @@ __precompile__(true)
 
 module DelimitedFiles
 
+using Mmap
+
 import Base: _default_delims, tryparse_internal, show
 
 export readdlm, writedlm

diff --git a/stdlib/Mmap/docs/src/index.md b/stdlib/Mmap/docs/src/index.md
@@ -0,0 +1,7 @@
+# Memory-mapped I/O
+
+```@docs
+Mmap.Anonymous
+Mmap.mmap
+Mmap.sync!
+```
diff --git a/base/mmap.jl → stdlib/Mmap/src/Mmap.jl b/base/mmap.jl → stdlib/Mmap/src/Mmap.jl
diff --git a/test/mmap.jl → stdlib/Mmap/test/runtests.jl b/test/mmap.jl → stdlib/Mmap/test/runtests.jl
@@ -1,5 +1,7 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
+using Test, Mmap
+
 file = tempname()
 write(file, "Hello World\n")
 t = b"Hello World"

diff --git a/stdlib/SharedArrays/docs/src/index.md b/stdlib/SharedArrays/docs/src/index.md
@@ -0,0 +1,9 @@
+# Shared Arrays
+
+```@docs
+SharedArrays.SharedArray
+SharedArrays.procs(::SharedArray)
+SharedArrays.sdata
+SharedArrays.indexpids
+SharedArrays.localindexes
+```
diff --git a/base/sharedarray.jl → stdlib/SharedArrays/src/SharedArrays.jl b/base/sharedarray.jl → stdlib/SharedArrays/src/SharedArrays.jl
@@ -1,7 +1,17 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-import .Serializer: serialize_cycle_header, serialize_type, writetag, UNDEFREF_TAG
-import .Distributed: RRID, procs
+module SharedArrays
+
+using Mmap, Base.Distributed
+
+import Base: length, size, ndims, IndexStyle, reshape, convert, deepcopy_internal, serialize, deserialize,
+             show, getindex, setindex!, fill!, rand!, similar, reduce, map!, copy!, unsafe_convert
+import Base.Random
+import Base.Serializer: serialize_cycle_header, serialize_type, writetag, UNDEFREF_TAG
+import Base.Distributed: RRID, procs
+import Base.Filesystem: JL_O_CREAT, JL_O_RDWR, S_IRUSR, S_IWUSR
+
+export SharedArray, SharedVector, SharedMatrix, sdata, indexpids, localindexes
 
 mutable struct SharedArray{T,N} <: DenseArray{T,N}
     id::RRID
@@ -663,6 +673,17 @@ end
 
 shm_unlink(shm_seg_name) = ccall(:shm_unlink, Cint, (Cstring,), shm_seg_name)
 shm_open(shm_seg_name, oflags, permissions) = ccall(:shm_open, Cint,
-    (Cstring, Cint, Cmode_t), shm_seg_name, oflags, permissions)
+    (Cstring, Cint, Base.Cmode_t), shm_seg_name, oflags, permissions)
 
 end # os-test
+
+# 0.7 deprecations
+
+@deprecate SharedArray(::Type{T}, dims::Dims{N}; kwargs...) where {T,N} SharedArray{T}(dims; kwargs...)
+@deprecate SharedArray(::Type{T}, dims::Int...; kwargs...) where {T}    SharedArray{T}(dims...; kwargs...)
+@deprecate(SharedArray(filename::AbstractString, ::Type{T}, dims::NTuple{N,Int}, offset; kwargs...) where {T,N},
+           SharedArray{T}(filename, dims, offset; kwargs...))
+@deprecate(SharedArray(filename::AbstractString, ::Type{T}, dims::NTuple, offset; kwargs...) where {T},
+           SharedArray{T}(filename, dims, offset; kwargs...))
+
+end # module