Backports for 1.0.3

.travis.yml

@@ -75,7 +75,7 @@ before_install:
         contrib/travis_fastfail.sh || exit 1;
         brew tap staticfloat/julia > /dev/null;
         brew rm --force $(brew deps --HEAD julia);
-        brew install -v gcc gmp mpfr pcre2 staticfloat/julia/openblas-julia staticfloat/julia/suite-sparse-julia staticfloat/juliadeps/libgfortran;
+        brew install -v ccache gcc gmp mpfr pcre2 staticfloat/julia/openblas-julia staticfloat/julia/suite-sparse-julia staticfloat/juliadeps/libgfortran;
         BUILDOPTS="-j3 USECLANG=1 USECCACHE=1 BINARYBUILDER_TRIPLET=x86_64-apple-darwin14 BINARYBUILDER_LLVM_ASSERTS=1";
         BUILDOPTS="$BUILDOPTS USE_BINARYBUILDER_LLVM=1 LLVM_CONFIG=$TRAVIS_BUILD_DIR/usr/tools/llvm-config LLVM_SIZE=$TRAVIS_BUILD_DIR/usr/tools/llvm-size";
         BUILDOPTS="$BUILDOPTS VERBOSE=1 USE_BLAS64=0 SUITESPARSE_INC=-I$(brew --prefix suite-sparse-julia)/include FORCE_ASSERTIONS=1";

Make.inc

@@ -1177,22 +1177,24 @@ $(subst /,\\,$(subst $(shell $(2) pwd),$(shell $(2) cmd //C cd),$(abspath $(1)))
 endef
 endif
 
-define symlink_target
-CLEAN_TARGETS += clean-$(2)/$(3)
+define symlink_target # (from, to-dir, to-name)
+CLEAN_TARGETS += clean-$$(abspath $(2)/$(3))
 clean-$$(abspath $(2)/$(3)):
 ifeq ($(BUILD_OS), WINNT)
-	@-cmd //C rmdir $$(call mingw_to_dos,$(2)/$(3),cd $(2) &&)
+	-cmd //C rmdir $$(call mingw_to_dos,$(2)/$(3),cd $(2) &&)
 else
-	@-rm $$(abspath $(2)/$(3))
+	-rm -r $$(abspath $(2)/$(3))
 endif
 $$(subst $$(abspath $(JULIAHOME))/,,$$(abspath $(2)/$(3))): $$(abspath $(2)/$(3))
 $$(abspath $(2)/$(3)): | $$(abspath $(2))
-ifeq ($(BUILD_OS), WINNT)
+ifeq ($$(BUILD_OS), WINNT)
 	@cmd //C mklink //J $$(call mingw_to_dos,$(2)/$(3),cd $(2) &&) $$(call mingw_to_dos,$(1),)
-else ifneq (,$(findstring CYGWIN,$(BUILD_OS)))
+else ifneq (,$$(findstring CYGWIN,$$(BUILD_OS)))
 	@cmd /C mklink /J $$(call cygpath_w,$(2)/$(3)) $$(call cygpath_w,$(1))
 else ifdef JULIA_VAGRANT_BUILD
-	@cp -R $$(abspath $(1)) $$@
+	@rm -r $$@
+	@cp -R $$(abspath $(1)) [email protected]
+	@mv [email protected] $$@
 else
 	@ln -sf $$(abspath $(1)) $$@
 endif

Makefile

@@ -37,7 +37,7 @@ configure:
 endif
 
 $(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir))))
-$(foreach link,base $(JULIAHOME)/test,$(eval $(call symlink_target,$(link),$(build_datarootdir)/julia,$(notdir $(link)))))
+$(foreach link,base $(JULIAHOME)/test,$(eval $(call symlink_target,$(link),$$(build_datarootdir)/julia,$(notdir $(link)))))
 
 julia_flisp.boot.inc.phony: julia-deps
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/src julia_flisp.boot.inc.phony
@@ -65,6 +65,9 @@ julia-base: julia-deps $(build_sysconfdir)/julia/startup.jl $(build_man1dir)/jul
 julia-libccalltest: julia-deps
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/src libccalltest
 
+julia-libllvmcalltest: julia-deps
+	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/src libllvmcalltest
+
 julia-src-release julia-src-debug : julia-src-% : julia-deps julia_flisp.boot.inc.phony
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/src libjulia-$*
 
@@ -80,7 +83,7 @@ julia-sysimg-release : julia-stdlib julia-sysimg julia-ui-release
 julia-sysimg-debug : julia-stdlib julia-sysimg julia-ui-debug
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) $(build_private_libdir)/sys-debug.$(SHLIB_EXT)
 
-julia-debug julia-release : julia-% : julia-ui-% julia-sysimg-% julia-symlink julia-libccalltest julia-base-cache
+julia-debug julia-release : julia-% : julia-ui-% julia-sysimg-% julia-symlink julia-libccalltest julia-libllvmcalltest julia-base-cache
 
 debug release : % : julia-%
 
@@ -180,10 +183,8 @@ COMPILER_SRCS := $(addprefix $(JULIAHOME)/, \
 COMPILER_SRCS += $(shell find $(JULIAHOME)/base/compiler -name \*.jl)
 # sort these to remove duplicates
 BASE_SRCS := $(sort $(shell find $(JULIAHOME)/base -name \*.jl -and -not -name sysimg.jl) \
-                    $(shell find $(BUILDROOT)/base -name \*.jl  -and -not -name sysimg.jl)) \
-                    $(shell find $(JULIAHOME)/stdlib/Base64/src -name \*.jl)
-STDLIB_SRCS := $(JULIAHOME)/base/sysimg.jl $(shell find $(JULIAHOME)/stdlib/*/src -name \*.jl)
-STDLIB_PKGS := $(shell cd $(JULIAHOME)/stdlib && ls)
+                    $(shell find $(BUILDROOT)/base -name \*.jl  -and -not -name sysimg.jl))
+STDLIB_SRCS := $(JULIAHOME)/base/sysimg.jl $(shell find $(build_datarootdir)/julia/stdlib/$(VERSDIR)/*/src -name \*.jl)
 RELBUILDROOT := $(shell $(JULIAHOME)/contrib/relative_path.sh "$(JULIAHOME)/base" "$(BUILDROOT)/base/")
 
 $(build_private_libdir)/corecompiler.ji: $(COMPILER_SRCS) | $(build_private_libdir)
@@ -230,7 +231,7 @@ JL_TARGETS += julia-debug
 endif
 
 # private libraries, that are installed in $(prefix)/lib/julia
-JL_PRIVATE_LIBS-0 := libccalltest
+JL_PRIVATE_LIBS-0 := libccalltest libllvmcalltest
 ifeq ($(USE_GPL_LIBS), 1)
 JL_PRIVATE_LIBS-0 += libsuitesparse_wrapper
 JL_PRIVATE_LIBS-$(USE_SYSTEM_SUITESPARSE) += libamd libcamd libccolamd libcholmod libcolamd libumfpack libspqr libsuitesparseconfig
@@ -381,6 +382,10 @@ endif
 	# Remove various files which should not be installed
 	-rm -f $(DESTDIR)$(datarootdir)/julia/base/version_git.sh
 	-rm -f $(DESTDIR)$(datarootdir)/julia/test/Makefile
+	-rm -f $(DESTDIR)$(datarootdir)/julia/stdlib/$(VERSDIR)/*/source-extracted
+	-rm -f $(DESTDIR)$(datarootdir)/julia/stdlib/$(VERSDIR)/*/build-configured
+	-rm -f $(DESTDIR)$(datarootdir)/julia/stdlib/$(VERSDIR)/*/build-compiled
+	-rm -f $(DESTDIR)$(datarootdir)/julia/stdlib/$(VERSDIR)/*/build-checked
 	# Copy in beautiful new man page
 	$(INSTALL_F) $(build_man1dir)/julia.1 $(DESTDIR)$(man1dir)/
 	# Copy icon and .desktop file
@@ -431,12 +436,9 @@ endif
 	mkdir -p $(DESTDIR)$(sysconfdir)
 	cp -R $(build_sysconfdir)/julia $(DESTDIR)$(sysconfdir)/
 
-distclean dist-clean:
+distclean:
 	-rm -fr $(BUILDROOT)/julia-*.tar.gz $(BUILDROOT)/julia*.exe $(BUILDROOT)/julia-*.7z $(BUILDROOT)/julia-$(JULIA_COMMIT)
 
-dist:
-	@echo \'dist\' target is deprecated: use \'binary-dist\' instead.
-
 binary-dist: distclean
 ifeq ($(USE_SYSTEM_BLAS),0)
 ifeq ($(ISX86),1)
@@ -511,16 +513,24 @@ endif
 
 	# Create file light-source-dist.tmp to hold all the filenames that go into the tarball
 	echo "base/version_git.jl" > light-source-dist.tmp
+
+	# Download all stdlibs and include the tarball filenames in light-source-dist.tmp
+	@$(MAKE) -C stdlib getall NO_GIT=1
+	-ls stdlib/srccache/*.tar.gz >> light-source-dist.tmp
+
 	# Exclude git, github and CI config files
 	git ls-files | sed -E -e '/^\..+/d' -e '/\/\..+/d' -e '/appveyor.yml/d' >> light-source-dist.tmp
 	find doc/_build/html >> light-source-dist.tmp
 
-# Make tarball with only Julia code
+# Make tarball with only Julia code + stdlib tarballs
 light-source-dist: light-source-dist.tmp
-	# Prefix everything with the current directory name (usually "julia"), then create tarball
-	DIRNAME=$$(basename $$(pwd)); \
-	sed -e "s_.*_$$DIRNAME/&_" light-source-dist.tmp > light-source-dist.tmp1; \
-	cd ../ && tar -cz --no-recursion -T $$DIRNAME/light-source-dist.tmp1 -f $$DIRNAME/julia-$(JULIA_VERSION)_$(JULIA_COMMIT).tar.gz
+	# Prefix everything with "julia-$(commit-sha)/" or "julia-$(version)/" and then create tarball
+	# To achieve prefixing, we temporarily create a symlink in the source directory that points back
+	# to the source directory.
+	sed -e "s_.*_julia-${JULIA_COMMIT}/&_" light-source-dist.tmp > light-source-dist.tmp1
+	ln -s . julia-${JULIA_COMMIT}
+	tar -cz --no-recursion -T light-source-dist.tmp1 -f julia-$(JULIA_VERSION)_$(JULIA_COMMIT).tar.gz
+	rm julia-${JULIA_COMMIT}
 
 source-dist:
 	@echo \'source-dist\' target is deprecated: use \'full-source-dist\' instead.
@@ -529,24 +539,26 @@ source-dist:
 full-source-dist: light-source-dist.tmp
 	# Get all the dependencies downloaded
 	@$(MAKE) -C deps getall NO_GIT=1
-	@$(MAKE) -C stdlib getall
 
 	# Create file full-source-dist.tmp to hold all the filenames that go into the tarball
 	cp light-source-dist.tmp full-source-dist.tmp
-	-ls deps/srccache/*.tar.gz deps/srccache/*.tar.bz2 deps/srccache/*.tar.xz deps/srccache/*.tgz deps/srccache/*.zip deps/srccache/*.pem stdlib/srccache/*.tar.gz >> full-source-dist.tmp
+	-ls deps/srccache/*.tar.gz deps/srccache/*.tar.bz2 deps/srccache/*.tar.xz deps/srccache/*.tgz deps/srccache/*.zip deps/srccache/*.pem >> full-source-dist.tmp
 
-	# Prefix everything with the current directory name (usually "julia"), then create tarball
-	DIRNAME=$$(basename $$(pwd)); \
-	sed -e "s_.*_$$DIRNAME/&_" full-source-dist.tmp > full-source-dist.tmp1; \
-	cd ../ && tar -cz --no-recursion -T $$DIRNAME/full-source-dist.tmp1 -f $$DIRNAME/julia-$(JULIA_VERSION)_$(JULIA_COMMIT)-full.tar.gz
+	# Prefix everything with "julia-$(commit-sha)/" or "julia-$(version)/" and then create tarball
+	# To achieve prefixing, we temporarily create a symlink in the source directory that points back
+	# to the source directory.
+	sed -e "s_.*_julia-${JULIA_COMMIT}/&_" full-source-dist.tmp > full-source-dist.tmp1
+	ln -s . julia-${JULIA_COMMIT}
+	tar -cz --no-recursion -T full-source-dist.tmp1 -f julia-$(JULIA_VERSION)_$(JULIA_COMMIT)-full.tar.gz
+	rm julia-${JULIA_COMMIT}
 
 clean: | $(CLEAN_TARGETS)
 	@-$(MAKE) -C $(BUILDROOT)/base clean
 	@-$(MAKE) -C $(BUILDROOT)/doc clean
 	@-$(MAKE) -C $(BUILDROOT)/src clean
 	@-$(MAKE) -C $(BUILDROOT)/ui clean
 	@-$(MAKE) -C $(BUILDROOT)/test clean
-	@-$(MAKE) -C $(BUILDROOT)/stdlib clean-pkg
+	@-$(MAKE) -C $(BUILDROOT)/stdlib clean
 	-rm -f $(BUILDROOT)/julia
 	-rm -f $(BUILDROOT)/*.tar.gz
 	-rm -f $(build_depsbindir)/stringreplace \
@@ -562,6 +574,7 @@ cleanall: clean
 	-rm -fr $(build_prefix) $(build_staging)
 
 distcleanall: cleanall
+	@-$(MAKE) -C $(BUILDROOT)/stdlib distclean
 	@-$(MAKE) -C $(BUILDROOT)/deps distcleanall
 	@-$(MAKE) -C $(BUILDROOT)/doc cleanall
 

base/bitarray.jl

@@ -1510,37 +1510,70 @@ function findprev(testf::Function, B::BitArray, start::Integer)
 end
 #findlast(testf::Function, B::BitArray) = findprev(testf, B, 1)  ## defined in array.jl
 
+# findall helper functions
+# Generic case (>2 dimensions)
+function allindices!(I, B::BitArray)
+    ind = first(keys(B))
+    for k = 1:length(B)
+        I[k] = ind
+        ind = nextind(B, ind)
+    end
+end
+
+# Optimized case for vector
+function allindices!(I, B::BitVector)
+    I[:] .= 1:length(B)
+end
+
+# Optimized case for matrix
+function allindices!(I, B::BitMatrix)
+    k = 1
+    for c = 1:size(B,2), r = 1:size(B,1)
+        I[k] = CartesianIndex(r, c)
+        k += 1
+    end
+end
+
+@inline _overflowind(i1, irest::Tuple{}, size) = (i1, irest)
+@inline function _overflowind(i1, irest, size)
+    i2 = irest[1]
+    while i1 > size[1]
+        i1 -= size[1]
+        i2 += 1
+    end
+    i2, irest = _overflowind(i2, tail(irest), tail(size))
+    return (i1, (i2, irest...))
+end
+
+@inline _toind(i1, irest::Tuple{}) = i1
+@inline _toind(i1, irest) = CartesianIndex(i1, irest...)
+
 function findall(B::BitArray)
-    l = length(B)
     nnzB = count(B)
-    ind = first(keys(B))
-    I = Vector{typeof(ind)}(undef, nnzB)
+    I = Vector{eltype(keys(B))}(undef, nnzB)
     nnzB == 0 && return I
+    nnzB == length(B) && (allindices!(I, B); return I)
     Bc = B.chunks
-    Icount = 1
-    for i = 1:length(Bc)-1
-        u = UInt64(1)
-        c = Bc[i]
-        for j = 1:64
-            if c & u != 0
-                I[Icount] = ind
-                Icount += 1
-            end
-            ind = nextind(B, ind)
-            u <<= 1
-        end
-    end
-    u = UInt64(1)
-    c = Bc[end]
-    for j = 0:_mod64(l-1)
-        if c & u != 0
-            I[Icount] = ind
-            Icount += 1
+    Bs = size(B)
+    Bi = i1 = i = 1
+    irest = ntuple(one, ndims(B) - 1)
+    c = Bc[1]
+    @inbounds while true
+        while c == 0
+            Bi == length(Bc) && return I
+            i1 += 64
+            Bi += 1
+            c = Bc[Bi]
         end
-        ind = nextind(B, ind)
-        u <<= 1
+
+        tz = trailing_zeros(c)
+        c = _blsr(c)
+
+        i1, irest = _overflowind(i1 + tz, irest, Bs)
+        I[i] = _toind(i1, irest)
+        i += 1
+        i1 -= tz
     end
-    return I
 end
 
 # For performance

base/broadcast.jl

@@ -436,11 +436,19 @@ end
 _bcs1(a::Integer, b::Integer) = a == 1 ? b : (b == 1 ? a : (a == b ? a : throw(DimensionMismatch("arrays could not be broadcast to a common size"))))
 _bcs1(a::Integer, b) = a == 1 ? b : (first(b) == 1 && last(b) == a ? b : throw(DimensionMismatch("arrays could not be broadcast to a common size")))
 _bcs1(a, b::Integer) = _bcs1(b, a)
-_bcs1(a, b) = _bcsm(b, a) ? b : (_bcsm(a, b) ? a : throw(DimensionMismatch("arrays could not be broadcast to a common size")))
+_bcs1(a, b) = _bcsm(b, a) ? _sametype(b, a) : (_bcsm(a, b) ? _sametype(a, b) : throw(DimensionMismatch("arrays could not be broadcast to a common size")))
 # _bcsm tests whether the second index is consistent with the first
 _bcsm(a, b) = a == b || length(b) == 1
 _bcsm(a, b::Number) = b == 1
 _bcsm(a::Number, b::Number) = a == b || b == 1
+# Ensure inferrability when dealing with axes of different AbstractUnitRange types
+# (We may not want to define general promotion rules between, say, OneTo and Slice, but if
+#  we get here we know the axes are at least consistent)
+_sametype(a::T, b::T) where T = a
+_sametype(a::OneTo, b::OneTo) = OneTo{Int}(a)
+_sametype(a::OneTo, b) = OneTo{Int}(a)
+_sametype(a, b::OneTo) = OneTo{Int}(a)
+_sametype(a, b) = UnitRange{Int}(a)
 
 ## Check that all arguments are broadcast compatible with shape
 # comparing one input against a shape

base/compiler/abstractinterpretation.jl

@@ -12,6 +12,10 @@ const _REF_NAME = Ref.body.name
 # logic #
 #########
 
+# see if the inference result might affect the final answer
+call_result_unused(frame::InferenceState, pc::LineNum=frame.currpc) =
+    isexpr(frame.src.code[frame.currpc], :call) && isempty(frame.ssavalue_uses[pc])
+
 function abstract_call_gf_by_type(@nospecialize(f), argtypes::Vector{Any}, @nospecialize(atype), sv::InferenceState)
     atype_params = unwrap_unionall(atype).parameters
     ft = unwrap_unionall(atype_params[1]) # TODO: ccall jl_first_argument_datatype here
@@ -95,6 +99,7 @@ function abstract_call_gf_by_type(@nospecialize(f), argtypes::Vector{Any}, @nosp
         rettype === Any && break
     end
     # try constant propagation if only 1 method is inferred to non-Bottom
+    # this is in preparation for inlining, or improving the return result
     if nonbot > 0 && seen == napplicable && !edgecycle && isa(rettype, Type) && sv.params.ipo_constant_propagation
         # if there's a possibility we could constant-propagate a better result
         # (hopefully without doing too much work), try to do that now
@@ -105,6 +110,15 @@ function abstract_call_gf_by_type(@nospecialize(f), argtypes::Vector{Any}, @nosp
             rettype = const_rettype
         end
     end
+    if call_result_unused(sv) && !(rettype === Bottom)
+        # We're mainly only here because the optimizer might want this code,
+        # but we ourselves locally don't typically care about it locally
+        # (beyond checking if it always throws).
+        # So avoid adding an edge, since we don't want to bother attempting
+        # to improve our result even if it does change (to always throw),
+        # and avoid keeping track of a more complex result type.
+        rettype = Any
+    end
     if !(rettype === Any) # adding a new method couldn't refine (widen) this type
         for edge in edges
             add_backedge!(edge::MethodInstance, sv)
@@ -232,6 +246,13 @@ function abstract_call_method(method::Method, @nospecialize(sig), sparams::Simpl
             if infstate.linfo.specTypes == sig
                 # avoid widening when detecting self-recursion
                 # TODO: merge call cycle and return right away
+                if call_result_unused(sv)
+                    # since we don't use the result (typically),
+                    # we have a self-cycle in the call-graph, but not in the inference graph (typically):
+                    # break this edge now (before we record it) by returning early
+                    # (non-typically, this means that we lose the ability to detect a guaranteed StackOverflow in some cases)
+                    return Any, false, nothing
+                end
                 topmost = nothing
                 edgecycle = true
                 break
@@ -299,9 +320,27 @@ function abstract_call_method(method::Method, @nospecialize(sig), sparams::Simpl
         if newsig !== sig
             # continue inference, but note that we've limited parameter complexity
             # on this call (to ensure convergence), so that we don't cache this result
+            if call_result_unused(sv)
+                # if we don't (typically) actually care about this result,
+                # don't bother trying to examine some complex abstract signature
+                # since it's very unlikely that we'll try to inline this,
+                # or want make an invoke edge to its calling convention return type.
+                # (non-typically, this means that we lose the ability to detect a guaranteed StackOverflow in some cases)
+                return Any, false, nothing
+            end
             infstate = sv
             topmost = topmost::InferenceState
-            while !(infstate.parent === topmost.parent)
+            while !(infstate === topmost.parent)
+                if call_result_unused(infstate)
+                    # If we won't propagate the result any further (since it's typically unused),
+                    # it's OK that we keep and cache the "limited" result in the parents
+                    # (non-typically, this means that we lose the ability to detect a guaranteed StackOverflow in some cases)
+                    # TODO: we might be able to halt progress much more strongly here,
+                    # since now we know we won't be able to keep anything much that we learned.
+                    # We were mainly only here to compute the calling convention return type,
+                    # but in most situations now, we are unlikely to be able to use that information.
+                    break
+                end
                 infstate.limited = true
                 for infstate_cycle in infstate.callers_in_cycle
                     infstate_cycle.limited = true
@@ -1080,7 +1119,9 @@ function typeinf_local(frame::InferenceState)
                     frame.bestguess = tmerge(frame.bestguess, rt)
                     for (caller, caller_pc) in frame.cycle_backedges
                         # notify backedges of updated type information
-                        if caller.stmt_types[caller_pc] !== ()
+                        typeassert(caller.stmt_types[caller_pc], VarTable) # we must have visited this statement before
+                        if !(caller.src.ssavaluetypes[caller_pc] === Any)
+                            # no reason to revisit if that call-site doesn't affect the final result
                             if caller_pc < caller.pc´´
                                 caller.pc´´ = caller_pc
                             end