stdlib-git: improve integration with git-external (#30075)

Previously some of the work was shifted, leading to awkward extraneous copies
and preventing usage of DEPS_GIT=1 mode (the main point of this script).
There also seemed to be unexpected failure modes in the current usage
where the files in stdlib/Pkg would routinely go missing for me. I don't know
the source of those, but I hope that this will fix it (because it makes more of
the state immutable).

Also noticed some issues with existing scripts (expanding variables at
unintended points, not rebuilding on some changes), and fixed those.

Make.inc

@@ -1181,22 +1181,24 @@ $(subst /,\\,$(subst $(shell $(2) pwd),$(shell $(2) cmd //C cd),$(abspath $(1)))
 endef
 endif
 
-define symlink_target
-CLEAN_TARGETS += clean-$(2)/$(3)
+define symlink_target # (from, to-dir, to-name)
+CLEAN_TARGETS += clean-$$(abspath $(2)/$(3))
 clean-$$(abspath $(2)/$(3)):
 ifeq ($(BUILD_OS), WINNT)
-	@-cmd //C rmdir $$(call mingw_to_dos,$(2)/$(3),cd $(2) &&)
+	-cmd //C rmdir $$(call mingw_to_dos,$(2)/$(3),cd $(2) &&)
 else
-	@-rm $$(abspath $(2)/$(3))
+	-rm -r $$(abspath $(2)/$(3))
 endif
 $$(subst $$(abspath $(JULIAHOME))/,,$$(abspath $(2)/$(3))): $$(abspath $(2)/$(3))
 $$(abspath $(2)/$(3)): | $$(abspath $(2))
-ifeq ($(BUILD_OS), WINNT)
+ifeq ($$(BUILD_OS), WINNT)
 	@cmd //C mklink //J $$(call mingw_to_dos,$(2)/$(3),cd $(2) &&) $$(call mingw_to_dos,$(1),)
-else ifneq (,$(findstring CYGWIN,$(BUILD_OS)))
+else ifneq (,$$(findstring CYGWIN,$$(BUILD_OS)))
 	@cmd /C mklink /J $$(call cygpath_w,$(2)/$(3)) $$(call cygpath_w,$(1))
 else ifdef JULIA_VAGRANT_BUILD
-	@cp -R $$(abspath $(1)) $$@
+	@rm -r $$@
+	@cp -R $$(abspath $(1)) $$@.tmp
+	@mv $$@.tmp $$@
 else
 	@ln -sf $$(abspath $(1)) $$@
 endif

Makefile

@@ -37,7 +37,7 @@ configure:
 endif
 
 $(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir))))
-$(foreach link,base $(JULIAHOME)/test,$(eval $(call symlink_target,$(link),$(build_datarootdir)/julia,$(notdir $(link)))))
+$(foreach link,base $(JULIAHOME)/test,$(eval $(call symlink_target,$(link),$$(build_datarootdir)/julia,$(notdir $(link)))))
 
 julia_flisp.boot.inc.phony: julia-deps
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/src julia_flisp.boot.inc.phony
@@ -183,10 +183,8 @@ COMPILER_SRCS := $(addprefix $(JULIAHOME)/, \
 COMPILER_SRCS += $(shell find $(JULIAHOME)/base/compiler -name \*.jl)
 # sort these to remove duplicates
 BASE_SRCS := $(sort $(shell find $(JULIAHOME)/base -name \*.jl -and -not -name sysimg.jl) \
-                    $(shell find $(BUILDROOT)/base -name \*.jl  -and -not -name sysimg.jl)) \
-                    $(shell find $(JULIAHOME)/stdlib/Base64/src -name \*.jl)
-STDLIB_SRCS := $(JULIAHOME)/base/sysimg.jl $(shell find $(JULIAHOME)/stdlib/*/src -name \*.jl)
-STDLIB_PKGS := $(shell cd $(JULIAHOME)/stdlib && ls)
+                    $(shell find $(BUILDROOT)/base -name \*.jl  -and -not -name sysimg.jl))
+STDLIB_SRCS := $(JULIAHOME)/base/sysimg.jl $(shell find $(build_datarootdir)/julia/stdlib/$(VERSDIR)/*/src -name \*.jl)
 RELBUILDROOT := $(shell $(JULIAHOME)/contrib/relative_path.sh "$(JULIAHOME)/base" "$(BUILDROOT)/base/")
 
 $(build_private_libdir)/corecompiler.ji: $(COMPILER_SRCS) | $(build_private_libdir)
@@ -397,6 +395,10 @@ endif
 	# Remove various files which should not be installed
 	-rm -f $(DESTDIR)$(datarootdir)/julia/base/version_git.sh
 	-rm -f $(DESTDIR)$(datarootdir)/julia/test/Makefile
+	-rm -f $(DESTDIR)$(datarootdir)/julia/stdlib/$(VERSDIR)/*/source-extracted
+	-rm -f $(DESTDIR)$(datarootdir)/julia/stdlib/$(VERSDIR)/*/build-configured
+	-rm -f $(DESTDIR)$(datarootdir)/julia/stdlib/$(VERSDIR)/*/build-compiled
+	-rm -f $(DESTDIR)$(datarootdir)/julia/stdlib/$(VERSDIR)/*/build-checked
 	# Copy in beautiful new man page
 	$(INSTALL_F) $(build_man1dir)/julia.1 $(DESTDIR)$(man1dir)/
 	# Copy icon and .desktop file
@@ -447,12 +449,9 @@ endif
 	mkdir -p $(DESTDIR)$(sysconfdir)
 	cp -R $(build_sysconfdir)/julia $(DESTDIR)$(sysconfdir)/
 
-distclean dist-clean:
+distclean:
 	-rm -fr $(BUILDROOT)/julia-*.tar.gz $(BUILDROOT)/julia*.exe $(BUILDROOT)/julia-*.7z $(BUILDROOT)/julia-$(JULIA_COMMIT)
 
-dist:
-	@echo \'dist\' target is deprecated: use \'binary-dist\' instead.
-
 binary-dist: distclean
 ifeq ($(USE_SYSTEM_BLAS),0)
 ifeq ($(ISX86),1)
@@ -518,7 +517,7 @@ endif
 	echo "base/version_git.jl" > light-source-dist.tmp
 
 	# Download all stdlibs and include the tarball filenames in light-source-dist.tmp
-	@$(MAKE) -C stdlib getall
+	@$(MAKE) -C stdlib getall NO_GIT=1
 	-ls stdlib/srccache/*.tar.gz >> light-source-dist.tmp
 
 	# Exclude git, github and CI config files
@@ -561,7 +560,7 @@ clean: | $(CLEAN_TARGETS)
 	@-$(MAKE) -C $(BUILDROOT)/src clean
 	@-$(MAKE) -C $(BUILDROOT)/ui clean
 	@-$(MAKE) -C $(BUILDROOT)/test clean
-	@-$(MAKE) -C $(BUILDROOT)/stdlib clean-pkg
+	@-$(MAKE) -C $(BUILDROOT)/stdlib clean
 	-rm -f $(BUILDROOT)/julia
 	-rm -f $(BUILDROOT)/*.tar.gz
 	-rm -f $(build_depsbindir)/stringreplace \
@@ -577,6 +576,7 @@ cleanall: clean
 	-rm -fr $(build_prefix) $(build_staging)
 
 distcleanall: cleanall
+	@-$(MAKE) -C $(BUILDROOT)/stdlib distclean
 	@-$(MAKE) -C $(BUILDROOT)/deps distcleanall
 	@-$(MAKE) -C $(BUILDROOT)/doc cleanall
 

base/sysimg.jl

@@ -423,11 +423,6 @@ include("loading.jl")
 # misc useful functions & macros
 include("util.jl")
 
-creating_sysimg = true
-# set up depot & load paths to be able to find stdlib packages
-init_depot_path()
-init_load_path()
-
 include("asyncmap.jl")
 
 include("multimedia.jl")
@@ -488,6 +483,11 @@ using .Base
 # Ensure this file is also tracked
 pushfirst!(Base._included_files, (@__MODULE__, joinpath(@__DIR__, "sysimg.jl")))
 
+# set up depot & load paths to be able to find stdlib packages
+@eval Base creating_sysimg = true
+Base.init_depot_path()
+Base.init_load_path()
+
 if Base.is_primary_base_module
 # load some stdlib packages but don't put their names in Main
 let

deps/checksums/Pkg-f9180e48b27a843aeee864db814dce57cb296b9b.tar.gz/md5

@@ -0,0 +1 @@
+42bdc87720b24e312c6882517f67d3a6

deps/checksums/Pkg-f9180e48b27a843aeee864db814dce57cb296b9b.tar.gz/sha512

@@ -0,0 +1 @@
+b2c3bc198e8dc8cc4c56f57d0f0f8dc6495fce7e0c5ca10414fc851a06a67775d4c6f42c76cd54a73a5e1d9c45a8f3de25be456a6a685da33738bcacce8ef232

deps/tools/common.mk

@@ -115,6 +115,7 @@ $(eval $(call dir_target,$(SRCCACHE)))
 
 upper = $(shell echo $1 | tr a-z A-Z)
 
+
 ## A rule for calling `make install` ##
 # example usage:
 #   $(call staged-install, \
@@ -177,6 +178,40 @@ $$(build_prefix)/manifest/$(strip $1): $$(build_staging)/$2.tgz | $(build_prefix
 	echo $2 > $$@
 endef
 
+
+## A rule for "installing" via a symlink ##
+# example usage:
+#   $(call symlink_install, \
+#       1 target, \               # name
+#       2 rel-build-directory, \  # BUILDDIR-relative path to content folder
+#       3 abs-target-directory)   # absolute path to installation folder for symlink `name`
+define symlink_install # (target-name, rel-from, abs-to)
+clean-$1: uninstall-$1
+install-$1: $$(build_prefix)/manifest/$1
+reinstall-$1: install-$1
+uninstall-$1:
+ifeq ($$(BUILD_OS), WINNT)
+	-cmd //C rmdir $$(call mingw_to_dos,$3/$1,cd $3 &&)
+else
+	-rm -r $3/$1
+endif
+	-rm $$(build_prefix)/manifest/$1
+
+$$(build_prefix)/manifest/$1: $$(BUILDDIR)/$2/build-compiled | $3 $$(build_prefix)/manifest
+	+[ ! \( -e $3/$1 -o -h $3/$1 \) ] || $$(MAKE) uninstall-$1
+ifeq ($$(BUILD_OS), WINNT)
+	cmd //C mklink //J $$(call mingw_to_dos,$3/$1,cd $3 &&) $$(call mingw_to_dos,$$(BUILDDIR)/$2,)
+else ifneq (,$$(findstring CYGWIN,$$(BUILD_OS)))
+	cmd /C mklink /J $$(call cygpath_w,$3/$1) $$(call cygpath_w,$$(BUILDDIR)/$2)
+else ifdef JULIA_VAGRANT_BUILD
+	cp -R $$(BUILDDIR)/$2 $3/$1
+else
+	ln -sf $$(abspath $$(BUILDDIR)/$2) $3/$1
+endif
+	echo $2 > $$@
+endef
+
+
 ifneq (bsdtar,$(findstring bsdtar,$(TAR_TEST)))
 #gnu tar
 UNTAR = $(TAR) -xzf

deps/tools/git-external.mk

@@ -21,9 +21,9 @@
 #   distclean-dirname:
 #
 define git-external
-include $(SRCDIR)/$1.version
+include $$(SRCDIR)/$1.version
 
-ifeq ($(DEPS_GIT),1)
+ifneq (,$$(filter $1 1,$$(DEPS_GIT)))
 $2_SRC_DIR := $1
 $2_SRC_FILE := $$(SRCCACHE)/$1.git
 $$($2_SRC_FILE)/HEAD: | $$(SRCCACHE)
@@ -53,20 +53,21 @@ $5/$1/source-compiled: $5/$1/.git/HEAD
 $$($2_SRC_FILE): | $$($2_SRC_FILE)/HEAD
 	touch -c $$@
 
-else # DEPS_GIT
+else # DEPS_GIT=0
 
 $2_SRC_DIR := $1-$$($2_SHA1)
 $2_SRC_FILE := $$(SRCCACHE)/$$($2_SRC_DIR).tar.gz
 $$($2_SRC_FILE): | $$(SRCCACHE)
 	$$(JLDOWNLOAD) $$@ $$(call $2_TAR_URL,$$($2_SHA1))
 $5/$$($2_SRC_DIR)/source-extracted: $$($2_SRC_FILE)
 	$$(JLCHECKSUM) $$<
-	-rm -r $$(dir $$@)
+	-[ ! \( -e $$(dir $$@) -o -h $$(dir $$@) \) ] || rm -r $$(dir $$@)
 	mkdir -p $$(dir $$@)
 	$(TAR) -C $$(dir $$@) --strip-components 1 -xf $$<
 	echo 1 > $$@
 endif # DEPS_GIT
 
+$$(build_prefix)/manifest/$1: $$(SRCDIR)/$1.version # make the manifest stale if the version file is touched (causing re-install for compliant targets)
 distclean-$1:
 	-rm -rf $5/$$($2_SRC_DIR) $$($2_SRC_FILE) $$(BUILDDIR)/$$($2_SRC_DIR)
 endef

doc/make.jl

@@ -22,7 +22,7 @@ cp_q(src, dest) = isfile(dest) || cp(src, dest)
 
 # make links for stdlib package docs, this is needed until #522 in Documenter.jl is finished
 const STDLIB_DOCS = []
-const STDLIB_DIR = joinpath(@__DIR__, "..", "stdlib")
+const STDLIB_DIR = Sys.STDLIB
 cd(joinpath(@__DIR__, "src")) do
     Base.rm("stdlib"; recursive=true, force=true)
     mkdir("stdlib")

stdlib/.gitignore

@@ -1,2 +1,3 @@
 /srccache
+/Pkg-*
 /Pkg

stdlib/Makefile

@@ -1,46 +1,47 @@
 default: install
 
-.PHONY: default extract-pkg get-pkg clean-pkg getall install
-
 SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
 JULIAHOME := $(abspath $(SRCDIR)/..)
 SRCCACHE := $(abspath $(SRCDIR)/srccache)
-BUILDDIR := $(SRCCACHE)
+BUILDDIR := .
 
 include $(JULIAHOME)/Make.inc
 include $(JULIAHOME)/deps/tools/common.mk
 include $(JULIAHOME)/deps/tools/git-external.mk
 
-VERSDIR := v`cut -d. -f1-2 < $(JULIAHOME)/VERSION`
+VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION)
 
-# Download and extract Pkg
-PKG := https://github.com/JuliaLang/Pkg.jl
-PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1
-$(eval $(call git-external,Pkg,PKG,,,$(BUILDDIR)))
-
-extract-pkg: $(BUILDDIR)/$(PKG_SRC_DIR)/source-extracted
-Pkg: $(BUILDDIR)/$(PKG_SRC_DIR)/source-extracted
-	rm -rf $@
-	cp -r $(BUILDDIR)/$(PKG_SRC_DIR) $@
-get-pkg: Pkg
-clean-pkg:
-	-rm -rf Pkg
-	-rm -rf $(BUILDDIR)/$(PKG_SRC_DIR)
+$(build_datarootdir)/julia/stdlib/$(VERSDIR):
+	mkdir -p $@
 
-# Generate symlinks to all stdlibs from usr/share/julia/stdlib/vX.Y/
 STDLIBS = Base64 CRC32c Dates DelimitedFiles Distributed FileWatching \
           Future InteractiveUtils Libdl LibGit2 LinearAlgebra Logging \
           Markdown Mmap Printf Profile Random REPL Serialization SHA \
           SharedArrays Sockets SparseArrays Statistics SuiteSparse Test Unicode UUIDs
+STDLIBS-EXT = Pkg
 
-$(foreach pkg, $(STDLIBS), $(eval $(call symlink_target,$(JULIAHOME)/stdlib/$(pkg),$(build_datarootdir)/julia/stdlib/$(shell echo $(VERSDIR)),$(pkg))))
-$(eval $(call symlink_target,$(BUILDROOT)/stdlib/Pkg,$(build_datarootdir)/julia/stdlib/$(shell echo $(VERSDIR)),Pkg))
-
-$(build_datarootdir)/julia/stdlib/$(shell echo $(VERSDIR)):
-	mkdir -p $@
-
-STDLIBS_LINK_TARGETS = $(addprefix $(build_datarootdir)/julia/stdlib/$(shell echo $(VERSDIR))/, $(STDLIBS) Pkg)
-
-install: Pkg $(STDLIBS_LINK_TARGETS)
-getall: get-pkg
-
+# Download and extract Pkg
+PKG_GIT_URL := git://github.com/JuliaLang/Pkg.jl.git
+PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1
+$(eval $(call git-external,Pkg,PKG,,,$(BUILDDIR)))
+$(BUILDDIR)/$(PKG_SRC_DIR)/build-compiled: $(BUILDDIR)/$(PKG_SRC_DIR)/source-extracted
+	@# no build steps
+	echo 1 > $@
+$(eval $(call symlink_install,Pkg,$$(PKG_SRC_DIR),$$(build_datarootdir)/julia/stdlib/$$(VERSDIR)))
+clean-Pkg:
+	-rm $(BUILDDIR)/$(PKG_SRC_DIR)/build-compiled
+get-Pkg: $(PKG_SRC_FILE)
+extract-Pkg: $(BUILDDIR)/$(PKG_SRC_DIR)/source-extracted
+configure-Pkg: extract-Pkg
+compile-Pkg: $(BUILDDIR)/$(PKG_SRC_DIR)/build-compiled
+
+
+# Generate symlinks to all stdlibs at usr/share/julia/stdlib/vX.Y/
+$(foreach module, $(STDLIBS), $(eval $(call symlink_target,$$(JULIAHOME)/stdlib/$(module),$$(build_datarootdir)/julia/stdlib/$$(VERSDIR),$(module))))
+
+STDLIBS_LINK_TARGETS := $(addprefix $(build_datarootdir)/julia/stdlib/$(VERSDIR)/,$(STDLIBS))
+
+getall get: get-Pkg
+install: install-Pkg $(STDLIBS_LINK_TARGETS)
+clean: clean-Pkg $(CLEAN_TARGETS)
+distclean: distclean-Pkg clean

stdlib/Pkg.version

@@ -1,2 +1,2 @@
 PKG_BRANCH = master
-PKG_SHA1 = d305e82fd353cb67e8a064800b9972ee1cb7b5e0
+PKG_SHA1 = f9180e48b27a843aeee864db814dce57cb296b9b