Merge pull request #17859 from JuliaLang/jn/dep-makefile-normalize

normalize deps Makefile targets and add uninstall

.travis.yml

@@ -42,7 +42,8 @@ matrix:
 cache:
   directories:
     - $TRAVIS_BUILD_DIR/deps/srccache
-    - $TRAVIS_BUILD_DIR/deps/build
+    - $TRAVIS_BUILD_DIR/deps/scratch
+    - $TRAVIS_BUILD_DIR/deps/usr-staging
 branches:
   only:
     - master
@@ -69,12 +70,14 @@ before_install:
         ln -s /usr/bin/gcc-5 $HOME/bin/x86_64-linux-gnu-gcc;
         ln -s /usr/bin/g++-5 $HOME/bin/x86_64-linux-gnu-g++;
         gcc --version;
+        BAR="bar -i 30";
         BUILDOPTS="-j3 VERBOSE=1 FORCE_ASSERTIONS=1 LLVM_ASSERTIONS=1";
         echo "override ARCH=$ARCH" >> Make.user;
         TESTSTORUN="all";
       elif [ `uname` = "Darwin" ]; then
         brew update;
         brew install -v jq bar;
+        BAR="bar";
         contrib/travis_fastfail.sh || exit 1;
         brew tap staticfloat/julia;
         brew rm --force $(brew deps --HEAD julia);
@@ -95,7 +98,20 @@ before_install:
 script:
     - make -C moreutils mispipe
     - make $BUILDOPTS -C base version_git.jl.phony
-    - moreutils/mispipe "make $BUILDOPTS NO_GIT=1 -C deps" bar > deps.log || cat deps.log
+    # capture the log, but only print it if `make deps` fails
+    # try to show the end of the log first, because this log might be very long (> 4MB)
+    # and thus be truncated by travis
+    - moreutils/mispipe "make $BUILDOPTS VERBOSE=0 -C deps 2> deps-err.log" "$BAR" > deps.log ||
+        { echo "-- deps build log stderr tail 100 --------------------------------------";
+          tail -n 100 deps-err.log;
+          echo "-- deps build log stdout tail 100 --------------------------------------";
+          tail -n 100 deps.log;
+          echo "-- deps build log stderr all -------------------------------------------";
+          cat deps-err.log;
+          echo "-- deps build log stdout all -------------------------------------------";
+          cat deps.log;
+          echo "-- end of deps build log -----------------------------------------------";
+          false; }
     - make $BUILDOPTS NO_GIT=1 prefix=/tmp/julia install | moreutils/ts -s "%.s"
     - make $BUILDOPTS NO_GIT=1 build-stats
     - du -sk /tmp/julia/*
@@ -112,9 +128,6 @@ script:
         /tmp/julia/bin/julia --check-bounds=yes runtests.jl $TESTSTORUN &&
         /tmp/julia/bin/julia --check-bounds=yes runtests.jl libgit2-online pkg
     - cd `dirname $TRAVIS_BUILD_DIR` && mv julia2 julia &&
-        rm -rf julia/deps/build/julia-env &&
-        rm -rf julia/deps/build/libssh2-*/CMakeFiles/Makefile2 &&
-        rm -rf julia/deps/build/curl-*/config.log &&
-        rm -rf julia/deps/build/curl-*/libtool
+        rm -rf julia/deps/scratch/julia-env
 # uncomment the following if failures are suspected to be due to the out-of-memory killer
 #    - dmesg

Make.inc

@@ -523,7 +523,7 @@ LDFLAGS += $(SANITIZE_LDFLAGS)
 #DEPS_CXXFLAGS += $(SANITIZE_OPTS)
 endif
 
-TAR=`which gtar 2>/dev/null || which tar 2>/dev/null`
+TAR := $(shell which gtar 2>/dev/null || which tar 2>/dev/null)
 TAR_TEST := $(shell $(TAR) --help 2>&1  | egrep 'bsdtar|strip-components')
 ifeq (,$(findstring components,$(TAR_TEST)))
 ifneq (bsdtar,$(findstring bsdtar,$(TAR_TEST)))
@@ -1004,9 +1004,8 @@ endif
 
 # ATLAS
 
-# ATLAS must have been previously built with "make -C deps compile-atlas" (without -jN),
-# or installed to usr/lib/libatlas from some another source (built as
-# a shared library, for your platform, single threaded)
+# ATLAS must have been previously  installed to usr/lib/libatlas
+# (built as a shared library, for your platform, single threaded)
 USE_ATLAS := 0
 ATLAS_LIBDIR := $(build_libdir)
 #or ATLAS_LIBDIR := /path/to/system/atlas/lib

contrib/check-whitespace.sh

@@ -16,6 +16,7 @@ file_patterns='
 *.scm
 *.inc
 *.make
+*.mk
 *.md
 *.rst
 *.sh

deps/.gitignore

@@ -1,2 +1,3 @@
 /srccache
 /build
+/scratch

deps/Makefile

@@ -1,19 +1,21 @@
 ## high-level setup ##
+default: install
 SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
 JULIAHOME := $(abspath $(SRCDIR)/..)
 ifeq ($(abspath .),$(abspath $(SRCDIR)))
-BUILDDIR := build
+BUILDDIR := scratch
 else
 BUILDDIR := .
 endif
 include $(SRCDIR)/Versions.make
 include $(JULIAHOME)/Make.inc
-include $(SRCDIR)/llvm-ver.make
+include $(SRCDIR)/tools/common.mk
+include $(SRCDIR)/tools/git-external.mk
 
 # Special comments:
 #
 # all targets in here should follow the same structure,
-# and provide a get-a, configure-a, compile-a, check-a, and install-a
+# and provide a get-a, extract-a, configure-a, compile-a, check-a, fastcheck-a, and install-a
 # additionally all targets should be listed in the getall target for easier off-line compilation
 # if you are adding a new target, it can help to copy an similar, existing target
 #
@@ -29,71 +31,18 @@ include $(SRCDIR)/llvm-ver.make
 # that all targets with a % in them on that line will
 # be rebuilt in a single invocation
 #
-
-## Some shared configuration options ##
-
-CONFIGURE_COMMON := --prefix=$(abspath $(build_prefix)) --build=$(BUILD_MACHINE) --libdir=$(abspath $(build_libdir)) --bindir=$(abspath $(build_depsbindir)) $(CUSTOM_LD_LIBRARY_PATH)
-ifneq ($(XC_HOST),)
-CONFIGURE_COMMON += --host=$(XC_HOST)
-endif
-ifeq ($(OS),WINNT)
-ifneq ($(USEMSVC), 1)
-CONFIGURE_COMMON += LDFLAGS="$(LDFLAGS) -Wl,--stack,8388608"
-endif
-endif
-CONFIGURE_COMMON += F77="$(FC)" CC="$(CC) $(DEPS_CFLAGS)" CXX="$(CXX) $(DEPS_CXXFLAGS)"
-
-CMAKE_CC_ARG := $(CC_ARG) $(DEPS_CFLAGS)
-CMAKE_CXX_ARG := $(CXX_ARG) $(DEPS_CXXFLAGS)
-
-CMAKE_COMMON := -DCMAKE_INSTALL_PREFIX:PATH=$(build_prefix) -DCMAKE_PREFIX_PATH=$(build_prefix)
-CMAKE_COMMON += -DCMAKE_INSTALL_LIBDIR=$(build_libdir) -DCMAKE_INSTALL_BINDIR=$(build_bindir)
-CMAKE_COMMON += -DLIB_INSTALL_DIR=$(build_shlibdir)
-ifneq ($(VERBOSE), 0)
-CMAKE_COMMON += -DCMAKE_VERBOSE_MAKEFILE=ON
-endif
-# The call to which here is to work around https://cmake.org/Bug/view.php?id=14366
-CMAKE_COMMON += -DCMAKE_C_COMPILER="$$(which $(CC_BASE))"
-ifneq ($(strip $(CMAKE_CC_ARG)),)
-CMAKE_COMMON += -DCMAKE_C_COMPILER_ARG1="$(CMAKE_CC_ARG)"
-endif
-CMAKE_COMMON += -DCMAKE_CXX_COMPILER="$(CXX_BASE)"
-ifneq ($(strip $(CMAKE_CXX_ARG)),)
-CMAKE_COMMON += -DCMAKE_CXX_COMPILER_ARG1="$(CMAKE_CXX_ARG)"
-endif
-
-ifeq ($(OS),WINNT)
-CMAKE_COMMON += -DCMAKE_SYSTEM_NAME=Windows
-ifneq ($(BUILD_OS),WINNT)
-CMAKE_COMMON += -DCMAKE_RC_COMPILER="$$(which $(CROSS_COMPILE)windres)"
-endif
-endif
-
-# For now this is LLVM specific, but I expect it won't be in the future
-ifeq ($(LLVM_USE_CMAKE),1)
-ifeq ($(CMAKE_GENERATOR),Ninja)
-CMAKE_GENERATOR_COMMAND := -G Ninja
-else ifeq ($(CMAKE_GENERATOR),make)
-CMAKE_GENERATOR_COMMAND := -G "Unix Makefiles"
-else
-$(error Unknown CMake generator '$(CMAKE_GENERATOR)'. Options are 'Ninja' and 'make')
-endif
-endif
-
-# If the top-level Makefile is called with environment variables,
-# they will override the values passed above to ./configure
-MAKE_COMMON := DESTDIR="" prefix=$(build_prefix) bindir=$(build_depsbindir) libdir=$(build_libdir) shlibdir=$(build_shlibdir) libexecdir=$(build_libexecdir) datarootdir=$(build_datarootdir) includedir=$(build_includedir) sysconfdir=$(build_sysconfdir) O=
-
+# to debug 'define' rules, replace eval at the usage site with info or error
+#
 
 ## Overall configuration of which rules exist and should be run by default ##
 
 # prevent installing libs into usr/lib64 on opensuse
 unexport CONFIG_SITE
 
-ifeq ($(USE_GPL_LIBS), 1)
-DEP_LIBS := suitesparse-wrapper
-else
 DEP_LIBS :=
+
+ifeq ($(USE_GPL_LIBS), 1)
+DEP_LIBS += suitesparse-wrapper
 endif
 
 ifeq ($(USE_SYSTEM_LIBUV), 0)
@@ -164,22 +113,8 @@ DEP_LIBS += gmp
 endif
 
 ifeq ($(USE_SYSTEM_LIBGIT2), 0)
-ifeq ($(USE_SYSTEM_MBEDTLS), 0)
-DEP_LIBS += mbedtls
-endif
-
-ifeq ($(USE_SYSTEM_LIBSSH2), 0)
-DEP_LIBS += libssh2
-endif
-
-ifneq ($(OS), WINNT)
-ifeq ($(USE_SYSTEM_CURL), 0)
-DEP_LIBS += curl
-endif
-endif
-
 DEP_LIBS += libgit2
-endif # USE_SYSTEM_LIBGIT2
+endif
 
 ifeq ($(USE_SYSTEM_MPFR), 0)
 DEP_LIBS += mpfr
@@ -209,54 +144,26 @@ DEP_LIBS += lapack
 endif
 endif
 
-#Platform specific flags
+DEP_LIBS_STAGED := $(filter-out suitesparse suitesparse-wrapper osxunwind virtualenv,$(DEP_LIBS)) # unlist targets that have not been converted to use the staged-install
 
-ifeq ($(OS), WINNT)
-LIBTOOL_CCLD := CCLD="$(CC) -no-undefined -avoid-version"
-endif
 
 ## Common build target prefixes
 
-default: install | $(build_prefix)
+default: | $(build_prefix)
 get: $(addprefix get-, $(DEP_LIBS))
+extract: $(addprefix extract-, $(DEP_LIBS))
 configure: $(addprefix configure-, $(DEP_LIBS))
 compile: $(addprefix compile-, $(DEP_LIBS))
 check: $(addprefix check-, $(DEP_LIBS))
+fastcheck: $(addprefix fastcheck-, $(DEP_LIBS))
+stage: $(addprefix stage-, $(DEP_LIBS_STAGED))
 install: $(addprefix install-, $(DEP_LIBS))
+uninstall: $(addprefix uninstall-, $(DEP_LIBS_STAGED))
 cleanall: $(addprefix clean-, $(DEP_LIBS))
 distcleanall: $(addprefix distclean-, $(DEP_LIBS))
 	rm -rf $(build_prefix)
 getall: get-llvm get-libuv get-pcre get-openlibm get-openspecfun get-dsfmt get-openblas get-lapack get-fftw get-suitesparse get-arpack get-unwind get-osxunwind get-gmp get-mpfr get-patchelf get-utf8proc get-virtualenv get-objconv get-mbedtls get-libssh2 get-curl get-libgit2
 
-## PATHS ##
-# sort is used to remove potential duplicates
-DIRS := $(sort $(build_bindir) $(build_depsbindir) $(build_libdir) $(build_includedir) $(build_sysconfdir) $(build_datarootdir) $(build_staging))
-
-$(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir))))
-
-$(build_prefix): | $(DIRS)
-$(eval $(call dir_target,$(SRCDIR)/srccache))
-
-## A rule for calling `make install` ##
-#	rule: dependencies
-#   	$(call make-install,rel-build-directory,add-args)
-#   	touch -c $@
-# this rule ensures that make install is more nearly atomic
-# so it's harder to get half-installed (or half-reinstalled) dependencies
-MAKE_DESTDIR = "$(build_staging)/$1"
-define staged-install
-	rm -rf $(build_staging)/$1
-	$2
-	mkdir -p $(build_prefix)
-	cp -af $(build_staging)/$1$(build_prefix)/* $(build_prefix)
-endef
-
-define make-install
-	$(call staged-install,$1,+$(MAKE) -C $(BUILDDIR)/$1 install $(MAKE_COMMON) $2 DESTDIR=$(call MAKE_DESTDIR,$1))
-endef
-
-include $(SRCDIR)/tools/git-external.mk
-
 include $(SRCDIR)/llvm.mk
 include $(SRCDIR)/libuv.mk
 include $(SRCDIR)/pcre.mk
@@ -278,9 +185,3 @@ include $(SRCDIR)/libssh2.mk
 include $(SRCDIR)/curl.mk
 include $(SRCDIR)/libgit2.mk
 include $(SRCDIR)/virtualenv.mk
-
-## phony targets ##
-
-.PHONY: default compile install cleanall distcleanall \
-	get-* configure-* compile-* check-* install-* \
-	clean-* distclean-* reinstall-* update-llvm