stdlib-git: improve integration with git-external

Make.inc

@@ -1181,22 +1181,24 @@ $(subst /,\\,$(subst $(shell $(2) pwd),$(shell $(2) cmd //C cd),$(abspath $(1)))
 endef
 endif
 
-define symlink_target
-CLEAN_TARGETS += clean-$(2)/$(3)
+define symlink_target # (from, to-dir, to-name)
+CLEAN_TARGETS += clean-$$(abspath $(2)/$(3))
 clean-$$(abspath $(2)/$(3)):
 ifeq ($(BUILD_OS), WINNT)
-	@-cmd //C rmdir $$(call mingw_to_dos,$(2)/$(3),cd $(2) &&)
+	-cmd //C rmdir $$(call mingw_to_dos,$(2)/$(3),cd $(2) &&)
 else
-	@-rm $$(abspath $(2)/$(3))
+	-rm -r $$(abspath $(2)/$(3))
 endif
 $$(subst $$(abspath $(JULIAHOME))/,,$$(abspath $(2)/$(3))): $$(abspath $(2)/$(3))
 $$(abspath $(2)/$(3)): | $$(abspath $(2))
-ifeq ($(BUILD_OS), WINNT)
+ifeq ($$(BUILD_OS), WINNT)
 	@cmd //C mklink //J $$(call mingw_to_dos,$(2)/$(3),cd $(2) &&) $$(call mingw_to_dos,$(1),)
-else ifneq (,$(findstring CYGWIN,$(BUILD_OS)))
+else ifneq (,$$(findstring CYGWIN,$$(BUILD_OS)))
 	@cmd /C mklink /J $$(call cygpath_w,$(2)/$(3)) $$(call cygpath_w,$(1))
 else ifdef JULIA_VAGRANT_BUILD
-	@cp -R $$(abspath $(1)) $$@
+	@rm -r $$@
+	@cp -R $$(abspath $(1)) [email protected]
+	@mv [email protected] $$@
 else
 	@ln -sf $$(abspath $(1)) $$@
 endif

Makefile

@@ -37,7 +37,7 @@ configure:
 endif
 
 $(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir))))
-$(foreach link,base $(JULIAHOME)/test,$(eval $(call symlink_target,$(link),$(build_datarootdir)/julia,$(notdir $(link)))))
+$(foreach link,base $(JULIAHOME)/test,$(eval $(call symlink_target,$(link),$$(build_datarootdir)/julia,$(notdir $(link)))))
 
 julia_flisp.boot.inc.phony: julia-deps
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/src julia_flisp.boot.inc.phony
@@ -180,10 +180,8 @@ COMPILER_SRCS := $(addprefix $(JULIAHOME)/, \
 COMPILER_SRCS += $(shell find $(JULIAHOME)/base/compiler -name \*.jl)
 # sort these to remove duplicates
 BASE_SRCS := $(sort $(shell find $(JULIAHOME)/base -name \*.jl -and -not -name sysimg.jl) \
-                    $(shell find $(BUILDROOT)/base -name \*.jl  -and -not -name sysimg.jl)) \
-                    $(shell find $(JULIAHOME)/stdlib/Base64/src -name \*.jl)
-STDLIB_SRCS := $(JULIAHOME)/base/sysimg.jl $(shell find $(JULIAHOME)/stdlib/*/src -name \*.jl)
-STDLIB_PKGS := $(shell cd $(JULIAHOME)/stdlib && ls)
+                    $(shell find $(BUILDROOT)/base -name \*.jl  -and -not -name sysimg.jl))
+STDLIB_SRCS := $(JULIAHOME)/base/sysimg.jl $(shell find $(build_datarootdir)/julia/stdlib/$(VERSDIR)/*/src -name \*.jl)
 RELBUILDROOT := $(shell $(JULIAHOME)/contrib/relative_path.sh "$(JULIAHOME)/base" "$(BUILDROOT)/base/")
 
 $(build_private_libdir)/corecompiler.ji: $(COMPILER_SRCS) | $(build_private_libdir)
@@ -394,6 +392,10 @@ endif
 	# Remove various files which should not be installed
 	-rm -f $(DESTDIR)$(datarootdir)/julia/base/version_git.sh
 	-rm -f $(DESTDIR)$(datarootdir)/julia/test/Makefile
+	-rm -f $(DESTDIR)$(datarootdir)/julia/stdlib/$(VERSDIR)/*/source-extracted
+	-rm -f $(DESTDIR)$(datarootdir)/julia/stdlib/$(VERSDIR)/*/build-configured
+	-rm -f $(DESTDIR)$(datarootdir)/julia/stdlib/$(VERSDIR)/*/build-compiled
+	-rm -f $(DESTDIR)$(datarootdir)/julia/stdlib/$(VERSDIR)/*/build-checked
 	# Copy in beautiful new man page
 	$(INSTALL_F) $(build_man1dir)/julia.1 $(DESTDIR)$(man1dir)/
 	# Copy icon and .desktop file
@@ -444,12 +446,9 @@ endif
 	mkdir -p $(DESTDIR)$(sysconfdir)
 	cp -R $(build_sysconfdir)/julia $(DESTDIR)$(sysconfdir)/
 
-distclean dist-clean:
+distclean:
 	-rm -fr $(BUILDROOT)/julia-*.tar.gz $(BUILDROOT)/julia*.exe $(BUILDROOT)/julia-*.7z $(BUILDROOT)/julia-$(JULIA_COMMIT)
 
-dist:
-	@echo \'dist\' target is deprecated: use \'binary-dist\' instead.
-
 binary-dist: distclean
 ifeq ($(USE_SYSTEM_BLAS),0)
 ifeq ($(ISX86),1)
@@ -519,7 +518,7 @@ endif
 	echo "base/version_git.jl" > light-source-dist.tmp
 
 	# Download all stdlibs and include the tarball filenames in light-source-dist.tmp
-	@$(MAKE) -C stdlib getall
+	@$(MAKE) -C stdlib getall NO_GIT=1
 	-ls stdlib/srccache/*.tar.gz >> light-source-dist.tmp
 
 	# Exclude git, github and CI config files
@@ -562,7 +561,7 @@ clean: | $(CLEAN_TARGETS)
 	@-$(MAKE) -C $(BUILDROOT)/src clean
 	@-$(MAKE) -C $(BUILDROOT)/ui clean
 	@-$(MAKE) -C $(BUILDROOT)/test clean
-	@-$(MAKE) -C $(BUILDROOT)/stdlib clean-pkg
+	@-$(MAKE) -C $(BUILDROOT)/stdlib clean
 	-rm -f $(BUILDROOT)/julia
 	-rm -f $(BUILDROOT)/*.tar.gz
 	-rm -f $(build_depsbindir)/stringreplace \
@@ -578,6 +577,7 @@ cleanall: clean
 	-rm -fr $(build_prefix) $(build_staging)
 
 distcleanall: cleanall
+	@-$(MAKE) -C $(BUILDROOT)/stdlib distclean
 	@-$(MAKE) -C $(BUILDROOT)/deps distcleanall
 	@-$(MAKE) -C $(BUILDROOT)/doc cleanall
 

base/sysimg.jl

@@ -423,11 +423,6 @@ include("loading.jl")
 # misc useful functions & macros
 include("util.jl")
 
-creating_sysimg = true
-# set up depot & load paths to be able to find stdlib packages
-init_depot_path()
-init_load_path()
-
 include("asyncmap.jl")
 
 include("multimedia.jl")
@@ -488,6 +483,11 @@ using .Base
 # Ensure this file is also tracked
 pushfirst!(Base._included_files, (@__MODULE__, joinpath(@__DIR__, "sysimg.jl")))
 
+# set up depot & load paths to be able to find stdlib packages
+@eval Base creating_sysimg = true
+Base.init_depot_path()
+Base.init_load_path()
+
 if Base.is_primary_base_module
 # load some stdlib packages but don't put their names in Main
 let

deps/checksums/Pkg-f9180e48b27a843aeee864db814dce57cb296b9b.tar.gz/md5

@@ -0,0 +1 @@
+42bdc87720b24e312c6882517f67d3a6

deps/checksums/Pkg-f9180e48b27a843aeee864db814dce57cb296b9b.tar.gz/sha512

@@ -0,0 +1 @@
+b2c3bc198e8dc8cc4c56f57d0f0f8dc6495fce7e0c5ca10414fc851a06a67775d4c6f42c76cd54a73a5e1d9c45a8f3de25be456a6a685da33738bcacce8ef232

deps/tools/common.mk

@@ -115,6 +115,7 @@ $(eval $(call dir_target,$(SRCCACHE)))
 
 upper = $(shell echo $1 | tr a-z A-Z)
 
+
 ## A rule for calling `make install` ##
 # example usage:
 #   $(call staged-install, \
@@ -177,6 +178,40 @@ $$(build_prefix)/manifest/$(strip $1): $$(build_staging)/$2.tgz | $(build_prefix
 	echo $2 > $$@
 endef
 
+
+## A rule for "installing" via a symlink ##
+# example usage:
+#   $(call symlink_install, \
+#       1 target, \               # name
+#       2 rel-build-directory, \  # BUILDDIR-relative path to content folder
+#       3 abs-target-directory)   # absolute path to installation folder for symlink `name`
+define symlink_install # (target-name, rel-from, abs-to)
+clean-$1: uninstall-$1
+install-$1: $$(build_prefix)/manifest/$1
+reinstall-$1: install-$1
+uninstall-$1:
+ifeq ($$(BUILD_OS), WINNT)
+	-cmd //C rmdir $$(call mingw_to_dos,$3/$1,cd $3 &&)
+else
+	-rm -r $3/$1
+endif
+	-rm $$(build_prefix)/manifest/$1
+
+$$(build_prefix)/manifest/$1: $$(BUILDDIR)/$2/build-compiled | $3 $$(build_prefix)/manifest
+	+[ ! \( -e $3/$1 -o -h $3/$1 \) ] || $$(MAKE) uninstall-$1
+ifeq ($$(BUILD_OS), WINNT)
+	cmd //C mklink //J $$(call mingw_to_dos,$3/$1,cd $3 &&) $$(call mingw_to_dos,$$(BUILDDIR)/$2,)
+else ifneq (,$$(findstring CYGWIN,$$(BUILD_OS)))
+	cmd /C mklink /J $$(call cygpath_w,$3/$1) $$(call cygpath_w,$$(BUILDDIR)/$2)
+else ifdef JULIA_VAGRANT_BUILD
+	cp -R $$(BUILDDIR)/$2 $3/$1
+else
+	ln -sf $$(abspath $$(BUILDDIR)/$2) $3/$1
+endif
+	echo $2 > $$@
+endef
+
+
 ifneq (bsdtar,$(findstring bsdtar,$(TAR_TEST)))
 #gnu tar
 UNTAR = $(TAR) -xzf

deps/tools/git-external.mk

@@ -21,9 +21,9 @@
 #   distclean-dirname:
 #
 define git-external
-include $(SRCDIR)/$1.version
+include $$(SRCDIR)/$1.version
 
-ifeq ($(DEPS_GIT),1)
+ifneq (,$$(filter $1 1,$$(DEPS_GIT)))
 $2_SRC_DIR := $1
 $2_SRC_FILE := $$(SRCCACHE)/$1.git
 $$($2_SRC_FILE)/HEAD: | $$(SRCCACHE)
@@ -53,20 +53,21 @@ $5/$1/source-compiled: $5/$1/.git/HEAD
 $$($2_SRC_FILE): | $$($2_SRC_FILE)/HEAD
 	touch -c $$@
 
-else # DEPS_GIT
+else # DEPS_GIT=0
 
 $2_SRC_DIR := $1-$$($2_SHA1)
 $2_SRC_FILE := $$(SRCCACHE)/$$($2_SRC_DIR).tar.gz
 $$($2_SRC_FILE): | $$(SRCCACHE)
 	$$(JLDOWNLOAD) $$@ $$(call $2_TAR_URL,$$($2_SHA1))
 $5/$$($2_SRC_DIR)/source-extracted: $$($2_SRC_FILE)
 	$$(JLCHECKSUM) $$<
-	-rm -r $$(dir $$@)
+	-[ ! \( -e $$(dir $$@) -o -h $$(dir $$@) \) ] || rm -r $$(dir $$@)
 	mkdir -p $$(dir $$@)
 	$(TAR) -C $$(dir $$@) --strip-components 1 -xf $$<
 	echo 1 > $$@
 endif # DEPS_GIT
 
+$$(build_prefix)/manifest/$1: $$(SRCDIR)/$1.version # make the manifest stale if the version file is touched (causing re-install for compliant targets)
 distclean-$1:
 	-rm -rf $5/$$($2_SRC_DIR) $$($2_SRC_FILE) $$(BUILDDIR)/$$($2_SRC_DIR)
 endef

doc/make.jl

@@ -22,7 +22,7 @@ cp_q(src, dest) = isfile(dest) || cp(src, dest)
 
 # make links for stdlib package docs, this is needed until #522 in Documenter.jl is finished
 const STDLIB_DOCS = []
-const STDLIB_DIR = joinpath(@__DIR__, "..", "stdlib")
+const STDLIB_DIR = Sys.STDLIB
 cd(joinpath(@__DIR__, "src")) do
     Base.rm("stdlib"; recursive=true, force=true)
     mkdir("stdlib")

stdlib/.gitignore

@@ -1,2 +1,3 @@
 /srccache
+/Pkg-*
 /Pkg

stdlib/Makefile

@@ -1,46 +1,47 @@
 default: install
 
-.PHONY: default extract-pkg get-pkg clean-pkg getall install
-
 SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
 JULIAHOME := $(abspath $(SRCDIR)/..)
 SRCCACHE := $(abspath $(SRCDIR)/srccache)
-BUILDDIR := $(SRCCACHE)
+BUILDDIR := .
 
 include $(JULIAHOME)/Make.inc
 include $(JULIAHOME)/deps/tools/common.mk
 include $(JULIAHOME)/deps/tools/git-external.mk
 
-VERSDIR := v`cut -d. -f1-2 < $(JULIAHOME)/VERSION`
+VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION)
 
-# Download and extract Pkg
-PKG := https://github.com/JuliaLang/Pkg.jl
-PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1
-$(eval $(call git-external,Pkg,PKG,,,$(BUILDDIR)))
-
-extract-pkg: $(BUILDDIR)/$(PKG_SRC_DIR)/source-extracted
-Pkg: $(BUILDDIR)/$(PKG_SRC_DIR)/source-extracted
-	rm -rf $@
-	cp -r $(BUILDDIR)/$(PKG_SRC_DIR) $@
-get-pkg: Pkg
-clean-pkg:
-	-rm -rf Pkg
-	-rm -rf $(BUILDDIR)/$(PKG_SRC_DIR)
+$(build_datarootdir)/julia/stdlib/$(VERSDIR):
+	mkdir -p $@
 
-# Generate symlinks to all stdlibs from usr/share/julia/stdlib/vX.Y/
 STDLIBS = Base64 CRC32c Dates DelimitedFiles Distributed FileWatching \
           Future InteractiveUtils Libdl LibGit2 LinearAlgebra Logging \
           Markdown Mmap Printf Profile Random REPL Serialization SHA \
           SharedArrays Sockets SparseArrays Statistics SuiteSparse Test Unicode UUIDs
+STDLIBS-EXT = Pkg
 
-$(foreach pkg, $(STDLIBS), $(eval $(call symlink_target,$(JULIAHOME)/stdlib/$(pkg),$(build_datarootdir)/julia/stdlib/$(shell echo $(VERSDIR)),$(pkg))))
-$(eval $(call symlink_target,$(BUILDROOT)/stdlib/Pkg,$(build_datarootdir)/julia/stdlib/$(shell echo $(VERSDIR)),Pkg))
-
-$(build_datarootdir)/julia/stdlib/$(shell echo $(VERSDIR)):
-	mkdir -p $@
-
-STDLIBS_LINK_TARGETS = $(addprefix $(build_datarootdir)/julia/stdlib/$(shell echo $(VERSDIR))/, $(STDLIBS) Pkg)
-
-install: Pkg $(STDLIBS_LINK_TARGETS)
-getall: get-pkg
-
+# Download and extract Pkg
+PKG_GIT_URL := git://github.com/JuliaLang/Pkg.jl.git
+PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1
+$(eval $(call git-external,Pkg,PKG,,,$(BUILDDIR)))
+$(BUILDDIR)/$(PKG_SRC_DIR)/build-compiled: $(BUILDDIR)/$(PKG_SRC_DIR)/source-extracted
+	@# no build steps
+	echo 1 > $@
+$(eval $(call symlink_install,Pkg,$$(PKG_SRC_DIR),$$(build_datarootdir)/julia/stdlib/$$(VERSDIR)))
+clean-Pkg:
+	-rm $(BUILDDIR)/$(PKG_SRC_DIR)/build-compiled
+get-Pkg: $(PKG_SRC_FILE)
+extract-Pkg: $(BUILDDIR)/$(PKG_SRC_DIR)/source-extracted
+configure-Pkg: extract-Pkg
+compile-Pkg: $(BUILDDIR)/$(PKG_SRC_DIR)/build-compiled
+
+
+# Generate symlinks to all stdlibs at usr/share/julia/stdlib/vX.Y/
+$(foreach module, $(STDLIBS), $(eval $(call symlink_target,$$(JULIAHOME)/stdlib/$(module),$$(build_datarootdir)/julia/stdlib/$$(VERSDIR),$(module))))
+
+STDLIBS_LINK_TARGETS := $(addprefix $(build_datarootdir)/julia/stdlib/$(VERSDIR)/,$(STDLIBS))
+
+getall get: get-Pkg
+install: install-Pkg $(STDLIBS_LINK_TARGETS)
+clean: clean-Pkg $(CLEAN_TARGETS)
+distclean: distclean-Pkg clean

stdlib/Pkg.version

@@ -1,2 +1,2 @@
 PKG_BRANCH = master
-PKG_SHA1 = d305e82fd353cb67e8a064800b9972ee1cb7b5e0
+PKG_SHA1 = f9180e48b27a843aeee864db814dce57cb296b9b