From fb3fabddbcf61ba899dc6bdfeebd43d4fb1caf49 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 26 Nov 2018 10:17:20 -0500 Subject: [PATCH] stdlib-git: improve integration with git-external (#30075) Previously some of the work was shifted, leading to awkward extraneous copies and preventing usage of DEPS_GIT=1 mode (the main point of this script). There also seemed to be unexpected failure modes in the current usage where the files in stdlib/Pkg would routinely go missing for me. I don't know the source of those, but I hope that this will fix it (because it makes more of the state immutable). Also noticed some issues with existing scripts (expanding variables at unintended points, not rebuilding on some changes), and fixed those. (cherry picked from commit ef50e2547d9a1d74f6b7356e6467d06b9fe7f8fb) --- Make.inc | 16 ++--- Makefile | 22 +++---- base/sysimg.jl | 10 ++-- .../md5 | 1 - .../sha512 | 1 - .../md5 | 1 + .../sha512 | 1 + deps/tools/common.mk | 35 +++++++++++ deps/tools/git-external.mk | 9 +-- doc/make.jl | 2 +- stdlib/.gitignore | 1 + stdlib/Makefile | 59 ++++++++++--------- stdlib/Pkg.version | 2 +- 13 files changed, 100 insertions(+), 60 deletions(-) delete mode 100644 deps/checksums/Pkg-d305e82fd353cb67e8a064800b9972ee1cb7b5e0.tar.gz/md5 delete mode 100644 deps/checksums/Pkg-d305e82fd353cb67e8a064800b9972ee1cb7b5e0.tar.gz/sha512 create mode 100644 deps/checksums/Pkg-f9180e48b27a843aeee864db814dce57cb296b9b.tar.gz/md5 create mode 100644 deps/checksums/Pkg-f9180e48b27a843aeee864db814dce57cb296b9b.tar.gz/sha512 diff --git a/Make.inc b/Make.inc index 00805eea63daaa..e5c102f9584ab7 100644 --- a/Make.inc +++ b/Make.inc @@ -1177,22 +1177,24 @@ $(subst /,\\,$(subst $(shell $(2) pwd),$(shell $(2) cmd //C cd),$(abspath $(1))) endef endif -define symlink_target -CLEAN_TARGETS += clean-$(2)/$(3) +define symlink_target # (from, to-dir, to-name) +CLEAN_TARGETS += clean-$$(abspath $(2)/$(3)) clean-$$(abspath $(2)/$(3)): ifeq ($(BUILD_OS), WINNT) - @-cmd //C rmdir $$(call mingw_to_dos,$(2)/$(3),cd $(2) &&) + -cmd //C rmdir $$(call mingw_to_dos,$(2)/$(3),cd $(2) &&) else - @-rm $$(abspath $(2)/$(3)) + -rm -r $$(abspath $(2)/$(3)) endif $$(subst $$(abspath $(JULIAHOME))/,,$$(abspath $(2)/$(3))): $$(abspath $(2)/$(3)) $$(abspath $(2)/$(3)): | $$(abspath $(2)) -ifeq ($(BUILD_OS), WINNT) +ifeq ($$(BUILD_OS), WINNT) @cmd //C mklink //J $$(call mingw_to_dos,$(2)/$(3),cd $(2) &&) $$(call mingw_to_dos,$(1),) -else ifneq (,$(findstring CYGWIN,$(BUILD_OS))) +else ifneq (,$$(findstring CYGWIN,$$(BUILD_OS))) @cmd /C mklink /J $$(call cygpath_w,$(2)/$(3)) $$(call cygpath_w,$(1)) else ifdef JULIA_VAGRANT_BUILD - @cp -R $$(abspath $(1)) $$@ + @rm -r $$@ + @cp -R $$(abspath $(1)) $$@.tmp + @mv $$@.tmp $$@ else @ln -sf $$(abspath $(1)) $$@ endif diff --git a/Makefile b/Makefile index 6aaa9aa97e0ec1..31206eeb5d9b81 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,7 @@ configure: endif $(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir)))) -$(foreach link,base $(JULIAHOME)/test,$(eval $(call symlink_target,$(link),$(build_datarootdir)/julia,$(notdir $(link))))) +$(foreach link,base $(JULIAHOME)/test,$(eval $(call symlink_target,$(link),$$(build_datarootdir)/julia,$(notdir $(link))))) julia_flisp.boot.inc.phony: julia-deps @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/src julia_flisp.boot.inc.phony @@ -183,10 +183,8 @@ COMPILER_SRCS := $(addprefix $(JULIAHOME)/, \ COMPILER_SRCS += $(shell find $(JULIAHOME)/base/compiler -name \*.jl) # sort these to remove duplicates BASE_SRCS := $(sort $(shell find $(JULIAHOME)/base -name \*.jl -and -not -name sysimg.jl) \ - $(shell find $(BUILDROOT)/base -name \*.jl -and -not -name sysimg.jl)) \ - $(shell find $(JULIAHOME)/stdlib/Base64/src -name \*.jl) -STDLIB_SRCS := $(JULIAHOME)/base/sysimg.jl $(shell find $(JULIAHOME)/stdlib/*/src -name \*.jl) -STDLIB_PKGS := $(shell cd $(JULIAHOME)/stdlib && ls) + $(shell find $(BUILDROOT)/base -name \*.jl -and -not -name sysimg.jl)) +STDLIB_SRCS := $(JULIAHOME)/base/sysimg.jl $(shell find $(build_datarootdir)/julia/stdlib/$(VERSDIR)/*/src -name \*.jl) RELBUILDROOT := $(shell $(JULIAHOME)/contrib/relative_path.sh "$(JULIAHOME)/base" "$(BUILDROOT)/base/") $(build_private_libdir)/corecompiler.ji: $(COMPILER_SRCS) | $(build_private_libdir) @@ -399,6 +397,10 @@ endif # Remove various files which should not be installed -rm -f $(DESTDIR)$(datarootdir)/julia/base/version_git.sh -rm -f $(DESTDIR)$(datarootdir)/julia/test/Makefile + -rm -f $(DESTDIR)$(datarootdir)/julia/stdlib/$(VERSDIR)/*/source-extracted + -rm -f $(DESTDIR)$(datarootdir)/julia/stdlib/$(VERSDIR)/*/build-configured + -rm -f $(DESTDIR)$(datarootdir)/julia/stdlib/$(VERSDIR)/*/build-compiled + -rm -f $(DESTDIR)$(datarootdir)/julia/stdlib/$(VERSDIR)/*/build-checked # Copy in beautiful new man page $(INSTALL_F) $(build_man1dir)/julia.1 $(DESTDIR)$(man1dir)/ # Copy icon and .desktop file @@ -449,12 +451,9 @@ endif mkdir -p $(DESTDIR)$(sysconfdir) cp -R $(build_sysconfdir)/julia $(DESTDIR)$(sysconfdir)/ -distclean dist-clean: +distclean: -rm -fr $(BUILDROOT)/julia-*.tar.gz $(BUILDROOT)/julia*.exe $(BUILDROOT)/julia-*.7z $(BUILDROOT)/julia-$(JULIA_COMMIT) -dist: - @echo \'dist\' target is deprecated: use \'binary-dist\' instead. - binary-dist: distclean ifeq ($(USE_SYSTEM_BLAS),0) ifeq ($(ISX86),1) @@ -524,7 +523,7 @@ endif echo "base/version_git.jl" > light-source-dist.tmp # Download all stdlibs and include the tarball filenames in light-source-dist.tmp - @$(MAKE) -C stdlib getall + @$(MAKE) -C stdlib getall NO_GIT=1 -ls stdlib/srccache/*.tar.gz >> light-source-dist.tmp # Exclude git, github and CI config files @@ -567,7 +566,7 @@ clean: | $(CLEAN_TARGETS) @-$(MAKE) -C $(BUILDROOT)/src clean @-$(MAKE) -C $(BUILDROOT)/ui clean @-$(MAKE) -C $(BUILDROOT)/test clean - @-$(MAKE) -C $(BUILDROOT)/stdlib clean-pkg + @-$(MAKE) -C $(BUILDROOT)/stdlib clean -rm -f $(BUILDROOT)/julia -rm -f $(BUILDROOT)/*.tar.gz -rm -f $(build_depsbindir)/stringreplace \ @@ -583,6 +582,7 @@ cleanall: clean -rm -fr $(build_prefix) $(build_staging) distcleanall: cleanall + @-$(MAKE) -C $(BUILDROOT)/stdlib distclean @-$(MAKE) -C $(BUILDROOT)/deps distcleanall @-$(MAKE) -C $(BUILDROOT)/doc cleanall diff --git a/base/sysimg.jl b/base/sysimg.jl index dd605b06417f7c..c1e23175b1b7af 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -422,11 +422,6 @@ include("loading.jl") # misc useful functions & macros include("util.jl") -creating_sysimg = true -# set up depot & load paths to be able to find stdlib packages -init_depot_path() -init_load_path() - include("asyncmap.jl") include("multimedia.jl") @@ -487,6 +482,11 @@ using .Base # Ensure this file is also tracked pushfirst!(Base._included_files, (@__MODULE__, joinpath(@__DIR__, "sysimg.jl"))) +# set up depot & load paths to be able to find stdlib packages +@eval Base creating_sysimg = true +Base.init_depot_path() +Base.init_load_path() + if Base.is_primary_base_module # load some stdlib packages but don't put their names in Main let diff --git a/deps/checksums/Pkg-d305e82fd353cb67e8a064800b9972ee1cb7b5e0.tar.gz/md5 b/deps/checksums/Pkg-d305e82fd353cb67e8a064800b9972ee1cb7b5e0.tar.gz/md5 deleted file mode 100644 index 15c30a29fdcc1e..00000000000000 --- a/deps/checksums/Pkg-d305e82fd353cb67e8a064800b9972ee1cb7b5e0.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -a633a375be6e5649865d8feee23bc177 diff --git a/deps/checksums/Pkg-d305e82fd353cb67e8a064800b9972ee1cb7b5e0.tar.gz/sha512 b/deps/checksums/Pkg-d305e82fd353cb67e8a064800b9972ee1cb7b5e0.tar.gz/sha512 deleted file mode 100644 index 34ef7a66b780c8..00000000000000 --- a/deps/checksums/Pkg-d305e82fd353cb67e8a064800b9972ee1cb7b5e0.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -9a16e4932017d0d159bcf65750eeeda1a637a51b6a1a4ebff957a81bc31fadc6db4be69426d6e99118129db9607e8b4d77df894c5df4fa55f88c45ebbb12d7b1 diff --git a/deps/checksums/Pkg-f9180e48b27a843aeee864db814dce57cb296b9b.tar.gz/md5 b/deps/checksums/Pkg-f9180e48b27a843aeee864db814dce57cb296b9b.tar.gz/md5 new file mode 100644 index 00000000000000..d8edfe8e3ca4e1 --- /dev/null +++ b/deps/checksums/Pkg-f9180e48b27a843aeee864db814dce57cb296b9b.tar.gz/md5 @@ -0,0 +1 @@ +42bdc87720b24e312c6882517f67d3a6 diff --git a/deps/checksums/Pkg-f9180e48b27a843aeee864db814dce57cb296b9b.tar.gz/sha512 b/deps/checksums/Pkg-f9180e48b27a843aeee864db814dce57cb296b9b.tar.gz/sha512 new file mode 100644 index 00000000000000..70c2fad19c1f8b --- /dev/null +++ b/deps/checksums/Pkg-f9180e48b27a843aeee864db814dce57cb296b9b.tar.gz/sha512 @@ -0,0 +1 @@ +b2c3bc198e8dc8cc4c56f57d0f0f8dc6495fce7e0c5ca10414fc851a06a67775d4c6f42c76cd54a73a5e1d9c45a8f3de25be456a6a685da33738bcacce8ef232 diff --git a/deps/tools/common.mk b/deps/tools/common.mk index a2d019d8c1e3e8..4030953db11d6d 100644 --- a/deps/tools/common.mk +++ b/deps/tools/common.mk @@ -115,6 +115,7 @@ $(eval $(call dir_target,$(SRCCACHE))) upper = $(shell echo $1 | tr a-z A-Z) + ## A rule for calling `make install` ## # example usage: # $(call staged-install, \ @@ -177,6 +178,40 @@ $$(build_prefix)/manifest/$(strip $1): $$(build_staging)/$2.tgz | $(build_prefix echo $2 > $$@ endef + +## A rule for "installing" via a symlink ## +# example usage: +# $(call symlink_install, \ +# 1 target, \ # name +# 2 rel-build-directory, \ # BUILDDIR-relative path to content folder +# 3 abs-target-directory) # absolute path to installation folder for symlink `name` +define symlink_install # (target-name, rel-from, abs-to) +clean-$1: uninstall-$1 +install-$1: $$(build_prefix)/manifest/$1 +reinstall-$1: install-$1 +uninstall-$1: +ifeq ($$(BUILD_OS), WINNT) + -cmd //C rmdir $$(call mingw_to_dos,$3/$1,cd $3 &&) +else + -rm -r $3/$1 +endif + -rm $$(build_prefix)/manifest/$1 + +$$(build_prefix)/manifest/$1: $$(BUILDDIR)/$2/build-compiled | $3 $$(build_prefix)/manifest + +[ ! \( -e $3/$1 -o -h $3/$1 \) ] || $$(MAKE) uninstall-$1 +ifeq ($$(BUILD_OS), WINNT) + cmd //C mklink //J $$(call mingw_to_dos,$3/$1,cd $3 &&) $$(call mingw_to_dos,$$(BUILDDIR)/$2,) +else ifneq (,$$(findstring CYGWIN,$$(BUILD_OS))) + cmd /C mklink /J $$(call cygpath_w,$3/$1) $$(call cygpath_w,$$(BUILDDIR)/$2) +else ifdef JULIA_VAGRANT_BUILD + cp -R $$(BUILDDIR)/$2 $3/$1 +else + ln -sf $$(abspath $$(BUILDDIR)/$2) $3/$1 +endif + echo $2 > $$@ +endef + + ifneq (bsdtar,$(findstring bsdtar,$(TAR_TEST))) #gnu tar UNTAR = $(TAR) -xzf diff --git a/deps/tools/git-external.mk b/deps/tools/git-external.mk index 152a895a5f39d9..df3e3df64fd0fb 100644 --- a/deps/tools/git-external.mk +++ b/deps/tools/git-external.mk @@ -21,9 +21,9 @@ # distclean-dirname: # define git-external -include $(SRCDIR)/$1.version +include $$(SRCDIR)/$1.version -ifeq ($(DEPS_GIT),1) +ifneq (,$$(filter $1 1,$$(DEPS_GIT))) $2_SRC_DIR := $1 $2_SRC_FILE := $$(SRCCACHE)/$1.git $$($2_SRC_FILE)/HEAD: | $$(SRCCACHE) @@ -53,7 +53,7 @@ $5/$1/source-compiled: $5/$1/.git/HEAD $$($2_SRC_FILE): | $$($2_SRC_FILE)/HEAD touch -c $$@ -else # DEPS_GIT +else # DEPS_GIT=0 $2_SRC_DIR := $1-$$($2_SHA1) $2_SRC_FILE := $$(SRCCACHE)/$$($2_SRC_DIR).tar.gz @@ -61,12 +61,13 @@ $$($2_SRC_FILE): | $$(SRCCACHE) $$(JLDOWNLOAD) $$@ $$(call $2_TAR_URL,$$($2_SHA1)) $5/$$($2_SRC_DIR)/source-extracted: $$($2_SRC_FILE) $$(JLCHECKSUM) $$< - -rm -r $$(dir $$@) + -[ ! \( -e $$(dir $$@) -o -h $$(dir $$@) \) ] || rm -r $$(dir $$@) mkdir -p $$(dir $$@) $(TAR) -C $$(dir $$@) --strip-components 1 -xf $$< echo 1 > $$@ endif # DEPS_GIT +$$(build_prefix)/manifest/$1: $$(SRCDIR)/$1.version # make the manifest stale if the version file is touched (causing re-install for compliant targets) distclean-$1: -rm -rf $5/$$($2_SRC_DIR) $$($2_SRC_FILE) $$(BUILDDIR)/$$($2_SRC_DIR) endef diff --git a/doc/make.jl b/doc/make.jl index d8884d94efc615..7a7e8cf1953dd7 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -22,7 +22,7 @@ cp_q(src, dest) = isfile(dest) || cp(src, dest) # make links for stdlib package docs, this is needed until #522 in Documenter.jl is finished const STDLIB_DOCS = [] -const STDLIB_DIR = joinpath(@__DIR__, "..", "stdlib") +const STDLIB_DIR = Sys.STDLIB cd(joinpath(@__DIR__, "src")) do Base.rm("stdlib"; recursive=true, force=true) mkdir("stdlib") diff --git a/stdlib/.gitignore b/stdlib/.gitignore index d8ce799be4bb97..a55686cd4ef258 100644 --- a/stdlib/.gitignore +++ b/stdlib/.gitignore @@ -1,2 +1,3 @@ /srccache +/Pkg-* /Pkg diff --git a/stdlib/Makefile b/stdlib/Makefile index 3c44b191fc1243..85c2a25c08d2a4 100644 --- a/stdlib/Makefile +++ b/stdlib/Makefile @@ -1,46 +1,47 @@ default: install -.PHONY: default extract-pkg get-pkg clean-pkg getall install - SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) JULIAHOME := $(abspath $(SRCDIR)/..) SRCCACHE := $(abspath $(SRCDIR)/srccache) -BUILDDIR := $(SRCCACHE) +BUILDDIR := . include $(JULIAHOME)/Make.inc include $(JULIAHOME)/deps/tools/common.mk include $(JULIAHOME)/deps/tools/git-external.mk -VERSDIR := v`cut -d. -f1-2 < $(JULIAHOME)/VERSION` +VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION) -# Download and extract Pkg -PKG := https://github.com/JuliaLang/Pkg.jl -PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 -$(eval $(call git-external,Pkg,PKG,,,$(BUILDDIR))) - -extract-pkg: $(BUILDDIR)/$(PKG_SRC_DIR)/source-extracted -Pkg: $(BUILDDIR)/$(PKG_SRC_DIR)/source-extracted - rm -rf $@ - cp -r $(BUILDDIR)/$(PKG_SRC_DIR) $@ -get-pkg: Pkg -clean-pkg: - -rm -rf Pkg - -rm -rf $(BUILDDIR)/$(PKG_SRC_DIR) +$(build_datarootdir)/julia/stdlib/$(VERSDIR): + mkdir -p $@ -# Generate symlinks to all stdlibs from usr/share/julia/stdlib/vX.Y/ STDLIBS = Base64 CRC32c Dates DelimitedFiles Distributed FileWatching \ Future InteractiveUtils Libdl LibGit2 LinearAlgebra Logging \ Markdown Mmap Printf Profile Random REPL Serialization SHA \ SharedArrays Sockets SparseArrays Statistics SuiteSparse Test Unicode UUIDs +STDLIBS-EXT = Pkg -$(foreach pkg, $(STDLIBS), $(eval $(call symlink_target,$(JULIAHOME)/stdlib/$(pkg),$(build_datarootdir)/julia/stdlib/$(shell echo $(VERSDIR)),$(pkg)))) -$(eval $(call symlink_target,$(BUILDROOT)/stdlib/Pkg,$(build_datarootdir)/julia/stdlib/$(shell echo $(VERSDIR)),Pkg)) - -$(build_datarootdir)/julia/stdlib/$(shell echo $(VERSDIR)): - mkdir -p $@ - -STDLIBS_LINK_TARGETS = $(addprefix $(build_datarootdir)/julia/stdlib/$(shell echo $(VERSDIR))/, $(STDLIBS) Pkg) - -install: Pkg $(STDLIBS_LINK_TARGETS) -getall: get-pkg - +# Download and extract Pkg +PKG_GIT_URL := git://github.com/JuliaLang/Pkg.jl.git +PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 +$(eval $(call git-external,Pkg,PKG,,,$(BUILDDIR))) +$(BUILDDIR)/$(PKG_SRC_DIR)/build-compiled: $(BUILDDIR)/$(PKG_SRC_DIR)/source-extracted + @# no build steps + echo 1 > $@ +$(eval $(call symlink_install,Pkg,$$(PKG_SRC_DIR),$$(build_datarootdir)/julia/stdlib/$$(VERSDIR))) +clean-Pkg: + -rm $(BUILDDIR)/$(PKG_SRC_DIR)/build-compiled +get-Pkg: $(PKG_SRC_FILE) +extract-Pkg: $(BUILDDIR)/$(PKG_SRC_DIR)/source-extracted +configure-Pkg: extract-Pkg +compile-Pkg: $(BUILDDIR)/$(PKG_SRC_DIR)/build-compiled + + +# Generate symlinks to all stdlibs at usr/share/julia/stdlib/vX.Y/ +$(foreach module, $(STDLIBS), $(eval $(call symlink_target,$$(JULIAHOME)/stdlib/$(module),$$(build_datarootdir)/julia/stdlib/$$(VERSDIR),$(module)))) + +STDLIBS_LINK_TARGETS := $(addprefix $(build_datarootdir)/julia/stdlib/$(VERSDIR)/,$(STDLIBS)) + +getall get: get-Pkg +install: install-Pkg $(STDLIBS_LINK_TARGETS) +clean: clean-Pkg $(CLEAN_TARGETS) +distclean: distclean-Pkg clean diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index 312a06e5365f75..9582ae5e2a884c 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,2 +1,2 @@ PKG_BRANCH = master -PKG_SHA1 = d305e82fd353cb67e8a064800b9972ee1cb7b5e0 +PKG_SHA1 = f9180e48b27a843aeee864db814dce57cb296b9b