From a434002b698ae6dd7cf26b3fac78d490b4a44f55 Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" Date: Wed, 3 Aug 2016 19:05:10 +0530 Subject: [PATCH 1/4] Fix pcre clean target. --- deps/pcre.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/pcre.mk b/deps/pcre.mk index c66197fc7f50c..5ae612452e763 100644 --- a/deps/pcre.mk +++ b/deps/pcre.mk @@ -36,7 +36,7 @@ $(PCRE_OBJ_TARGET): $(PCRE_SRC_TARGET) touch -c $@ clean-pcre: - -$(MAKE) -C pcre2-$(PCRE_VER) clean + -$(MAKE) -C $(BUILDDIR)/pcre2-$(PCRE_VER) clean -rm -f $(build_shlibdir)/libpcre* distclean-pcre: -rm -rf $(SRCDIR)/srccache/pcre2-$(PCRE_VER).tar.bz2 $(SRCDIR)/srccache/pcre2-$(PCRE_VER) $(BUILDDIR)/pcre2-$(PCRE_VER) From 11a8e07d89d26fe5c42a3cbcfbfbcf85a5d790aa Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" Date: Sat, 6 Aug 2016 14:32:52 +0530 Subject: [PATCH 2/4] Fix mbedtls clean target to delete all installed files. --- deps/mbedtls.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/mbedtls.mk b/deps/mbedtls.mk index 79c8513a44579..9ea8181408a7d 100644 --- a/deps/mbedtls.mk +++ b/deps/mbedtls.mk @@ -71,7 +71,7 @@ endif clean-mbedtls: -$(MAKE) -C $(BUILDDIR)/mbedtls-$(MBEDTLS_VER) clean - -rm -f $(MBEDTLS_OBJ_TARGET) + -rm -f $(MBEDTLS_OBJ_TARGET) $(build_shlibdir)/libmbed* distclean-mbedtls: -rm -rf $(SRCDIR)/srccache/$(MBEDTLS_SRC).tgz \ From 046c6eb7f896d806de3fc38da2e8c9153b2dba39 Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" Date: Wed, 3 Aug 2016 19:07:09 +0530 Subject: [PATCH 3/4] Build curl to provide proxy support for Pkg. The curl build depends on mbedtls and libssh2 Patch for mbedtls to allow curl to build with mbedtls support Add curl to LICENSE.md, README.md and all the other various locations --- LICENSE.md | 1 + Make.inc | 1 + Makefile | 3 ++ README.md | 1 + deps/Makefile | 9 ++++- deps/Versions.make | 1 + deps/checksums/curl-7.50.1.tar.bz2/md5 | 1 + deps/checksums/curl-7.50.1.tar.bz2/sha512 | 1 + deps/curl.mk | 46 +++++++++++++++++++++++ deps/mbedtls.mk | 6 ++- deps/patches/mbedtls-ssl.h.patch | 12 ++++++ 11 files changed, 79 insertions(+), 3 deletions(-) create mode 100644 deps/checksums/curl-7.50.1.tar.bz2/md5 create mode 100644 deps/checksums/curl-7.50.1.tar.bz2/sha512 create mode 100644 deps/curl.mk create mode 100644 deps/patches/mbedtls-ssl.h.patch diff --git a/LICENSE.md b/LICENSE.md index fd312fcee06f3..ab11841e04e27 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -57,6 +57,7 @@ their own licenses: - [FFTW](http://fftw.org/doc/License-and-Copyright.html) [GPL2+] - [GMP](http://gmplib.org/manual/Copying.html#Copying) [LGPL3+ or GPL2+] - [LIBGIT2](https://github.com/libgit2/libgit2/blob/development/COPYING) [GPL2+ with unlimited linking exception] +- [CURL](https://curl.haxx.se/docs/copyright.html) [MIT/X derivative] - [LIBSSH2](https://github.com/libssh2/libssh2/blob/master/COPYING) [BSD-3] - [MBEDTLS](https://tls.mbed.org/how-to-get) [either GPLv2 or Apache 2.0] - [MPFR](http://www.mpfr.org/mpfr-current/mpfr.html#Copying) [LGPL3+] diff --git a/Make.inc b/Make.inc index 8bd586dd92cb8..568a94cb22cf7 100644 --- a/Make.inc +++ b/Make.inc @@ -38,6 +38,7 @@ USE_SYSTEM_LIBUV:=0 USE_SYSTEM_UTF8PROC:=0 USE_SYSTEM_MBEDTLS:=0 USE_SYSTEM_LIBSSH2:=0 +USE_SYSTEM_CURL:=0 USE_SYSTEM_LIBGIT2:=0 USE_SYSTEM_PATCHELF:=0 diff --git a/Makefile b/Makefile index e978824d0ff96..7f0ef928f902f 100644 --- a/Makefile +++ b/Makefile @@ -277,6 +277,9 @@ endif ifeq ($(USE_SYSTEM_LIBSSH2),0) JL_PRIVATE_LIBS += ssh2 endif +ifeq ($(USE_SYSTEM_CURL),0) +JL_PRIVATE_LIBS += curl +endif ifeq ($(USE_SYSTEM_LIBGIT2),0) JL_PRIVATE_LIBS += git2 endif diff --git a/README.md b/README.md index 87aa47027db59..b33d164073069 100644 --- a/README.md +++ b/README.md @@ -283,6 +283,7 @@ Julia uses the following external libraries, which are automatically downloaded - **[GMP]** (>= 5.0) — GNU multiple precision arithmetic library, needed for `BigInt` support. - **[MPFR]** (>= 3.0) — GNU multiple precision floating point library, needed for arbitrary precision floating point (`BigFloat`) support. - **[libgit2]** (>= 0.23) — Git linkable library, used by Julia's package manager +- **[curl]** (>= 7.50) — libcurl provides download and proxy support for Julia's package manager - **[libssh2]** (>= 1.7) — library for SSH transport, used by libgit2 for packages with SSH remotes - **[mbedtls]** (>= 2.2) — library used for cryptography and transport layer security, used by libssh2 - **[utf8proc]** (>= 2.0) — a library for processing UTF-8 encoded Unicode strings diff --git a/deps/Makefile b/deps/Makefile index 08c3f96fc503e..9d6dd32375950 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -17,7 +17,7 @@ include $(SRCDIR)/llvm-ver.make # additionally all targets should be listed in the getall target for easier off-line compilation # if you are adding a new target, it can help to copy an similar, existing target # -# autoconf configure-driven scripts: llvm pcre arpack fftw unwind gmp mpfr patchelf libuv +# autoconf configure-driven scripts: llvm pcre arpack fftw unwind gmp mpfr patchelf libuv curl # custom Makefile rules: openlibm dsfmt suitesparse-wrapper suitesparse lapack openblas utf8proc objconv osxunwind # entirely custom: virtualenv # CMake libs: libgit2 libssh2 mbedtls @@ -171,6 +171,10 @@ ifeq ($(USE_SYSTEM_LIBSSH2), 0) STAGE2_DEPS += libssh2 endif +ifeq ($(USE_SYSTEM_CURL), 0) +STAGE2_DEPS += curl +endif + ifeq ($(USE_SYSTEM_LIBGIT2), 0) STAGE3_DEPS += libgit2 endif @@ -222,7 +226,7 @@ install: $(addprefix install-, $(DEP_LIBS)) cleanall: $(addprefix clean-, $(DEP_LIBS)) distcleanall: $(addprefix distclean-, $(DEP_LIBS)) rm -rf $(build_prefix) -getall: get-llvm get-libuv get-pcre get-openlibm get-openspecfun get-dsfmt get-openblas get-lapack get-fftw get-suitesparse get-arpack get-unwind get-osxunwind get-gmp get-mpfr get-patchelf get-utf8proc get-virtualenv get-objconv get-mbedtls get-libssh2 get-libgit2 +getall: get-llvm get-libuv get-pcre get-openlibm get-openspecfun get-dsfmt get-openblas get-lapack get-fftw get-suitesparse get-arpack get-unwind get-osxunwind get-gmp get-mpfr get-patchelf get-utf8proc get-virtualenv get-objconv get-mbedtls get-libssh2 get-curl get-libgit2 ## PATHS ## # sort is used to remove potential duplicates @@ -271,6 +275,7 @@ include $(SRCDIR)/mpfr.mk include $(SRCDIR)/patchelf.mk include $(SRCDIR)/mbedtls.mk include $(SRCDIR)/libssh2.mk +include $(SRCDIR)/curl.mk include $(SRCDIR)/libgit2.mk include $(SRCDIR)/virtualenv.mk diff --git a/deps/Versions.make b/deps/Versions.make index 4d6c5fc751b79..cdf2c1eb84de9 100644 --- a/deps/Versions.make +++ b/deps/Versions.make @@ -13,3 +13,4 @@ MPFR_VER = 3.1.4 PATCHELF_VER = 0.9 VIRTUALENV_VER = 15.0.0 MBEDTLS_VER = 2.3.0 +CURL_VER = 7.50.1 diff --git a/deps/checksums/curl-7.50.1.tar.bz2/md5 b/deps/checksums/curl-7.50.1.tar.bz2/md5 new file mode 100644 index 0000000000000..111b575728535 --- /dev/null +++ b/deps/checksums/curl-7.50.1.tar.bz2/md5 @@ -0,0 +1 @@ +015f6a0217ca6f2c5442ca406476920b diff --git a/deps/checksums/curl-7.50.1.tar.bz2/sha512 b/deps/checksums/curl-7.50.1.tar.bz2/sha512 new file mode 100644 index 0000000000000..95a02a7febe1c --- /dev/null +++ b/deps/checksums/curl-7.50.1.tar.bz2/sha512 @@ -0,0 +1 @@ +94acd91fcf8ff2605e1ba2d086c2c366257b61eaf516b9ea44e574e315feb5b30f6e47d89051f259e026ef5dd9edde5f7b15a6af9ee6a38f641da354e1e677b1 diff --git a/deps/curl.mk b/deps/curl.mk new file mode 100644 index 0000000000000..12295cde6ad08 --- /dev/null +++ b/deps/curl.mk @@ -0,0 +1,46 @@ +## CURL ## + +CURL_SRC_TARGET := $(BUILDDIR)/curl-$(CURL_VER)/.libs/libcurl.$(SHLIB_EXT) +CURL_OBJ_TARGET := $(build_shlibdir)/libcurl.$(SHLIB_EXT) + +ifneq ($(OS),WINNT) +CURL_LDFLAGS := $(RPATH_ESCAPED_ORIGIN) +endif + +$(SRCDIR)/srccache/curl-$(CURL_VER).tar.bz2: | $(SRCDIR)/srccache + $(JLDOWNLOAD) $@ https://curl.haxx.se/download/curl-$(CURL_VER).tar.bz2 +$(SRCDIR)/srccache/curl-$(CURL_VER)/configure: $(SRCDIR)/srccache/curl-$(CURL_VER).tar.bz2 $(MBEDTLS_OBJ_TARGET) $(LIBSSH2_OBJ_TARGET) + $(JLCHECKSUM) $< + cd $(dir $<) && $(TAR) jxf $(notdir $<) + touch -c $@ +$(BUILDDIR)/curl-$(CURL_VER)/config.status: $(SRCDIR)/srccache/curl-$(CURL_VER)/configure + mkdir -p $(dir $@) + cd $(dir $@) && \ + $< $(CONFIGURE_COMMON) --includedir=$(build_includedir) --with-mbedtls=$(build_shlibdir)/.. CFLAGS="$(CFLAGS) $(CURL_CFLAGS)" LDFLAGS="$(LDFLAGS) $(CURL_LDFLAGS)" + touch -c $@ +$(CURL_SRC_TARGET): $(BUILDDIR)/curl-$(CURL_VER)/config.status + $(MAKE) -C $(dir $<) $(LIBTOOL_CCLD) + touch -c $@ +$(BUILDDIR)/curl-$(CURL_VER)/checked: $(CURL_SRC_TARGET) +ifeq ($(OS),$(BUILD_OS)) +ifneq ($(OS),WINNT) + $(MAKE) -C $(dir $@) check -j1 +endif +endif + echo 1 > $@ +$(CURL_OBJ_TARGET): $(CURL_SRC_TARGET) + $(call make-install,curl-$(CURL_VER),$(LIBTOOL_CCLD)) + $(INSTALL_NAME_CMD)libcurl.$(SHLIB_EXT) $@ + touch -c $@ + +clean-curl: + -$(MAKE) -C $(BUILDDIR)/curl-$(CURL_VER) clean + -rm -f $(build_shlibdir)/libcurl* +distclean-curl: + -rm -rf $(SRCDIR)/srccache/curl-$(CURL_VER).tar.bz2 $(SRCDIR)/srccache/curl-$(CURL_VER) $(BUILDDIR)/curl-$(CURL_VER) + +get-curl: $(SRCDIR)/srccache/curl-$(CURL_VER).tar.bz2 +configure-curl: $(BUILDDIR)/curl-$(CURL_VER)/config.status +compile-curl: $(CURL_SRC_TARGET) +check-curl: $(BUILDDIR)/curl-$(CURL_VER)/checked +install-curl: $(CURL_OBJ_TARGET) diff --git a/deps/mbedtls.mk b/deps/mbedtls.mk index 9ea8181408a7d..ce0396a80f895 100644 --- a/deps/mbedtls.mk +++ b/deps/mbedtls.mk @@ -39,7 +39,11 @@ $(SRCDIR)/srccache/$(MBEDTLS_SRC)/mbedtls-config.patch-applied: | $(SRCDIR)/srcc cd $(SRCDIR)/srccache/$(MBEDTLS_SRC) && patch -p0 -f < $(SRCDIR)/patches/mbedtls-config.patch echo 1 > $@ -$(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/Makefile: $(SRCDIR)/srccache/$(MBEDTLS_SRC)/CMakeLists.txt $(SRCDIR)/srccache/$(MBEDTLS_SRC)/mbedtls-config.patch-applied +$(SRCDIR)/srccache/$(MBEDTLS_SRC)/mbedtls-ssl.h.patch-applied: | $(SRCDIR)/srccache/$(MBEDTLS_SRC)/CMakeLists.txt + cd $(SRCDIR)/srccache/$(MBEDTLS_SRC)/include/mbedtls && patch -p0 -f < $(SRCDIR)/patches/mbedtls-ssl.h.patch + echo 1 > $@ + +$(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/Makefile: $(SRCDIR)/srccache/$(MBEDTLS_SRC)/CMakeLists.txt $(SRCDIR)/srccache/$(MBEDTLS_SRC)/mbedtls-config.patch-applied $(SRCDIR)/srccache/$(MBEDTLS_SRC)/mbedtls-ssl.h.patch-applied mkdir -p $(dir $@) cd $(dir $@) && \ $(CMAKE) $(dir $<) $(MBEDTLS_OPTS) diff --git a/deps/patches/mbedtls-ssl.h.patch b/deps/patches/mbedtls-ssl.h.patch new file mode 100644 index 0000000000000..11785c5d88c60 --- /dev/null +++ b/deps/patches/mbedtls-ssl.h.patch @@ -0,0 +1,12 @@ +--- ssl.h.old 2016-06-28 18:12:06.000000000 +0530 ++++ ssl.h 2016-08-03 18:51:34.000000000 +0530 +@@ -54,7 +54,8 @@ + #endif + + #if defined(MBEDTLS_HAVE_TIME) +-#include ++//#include ++#include "platform.h" + #endif + + /* From 3687527bfc98be8438d2bf018e87c70ea123403b Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" Date: Wed, 3 Aug 2016 19:38:25 +0530 Subject: [PATCH 4/4] Have libgit2 use our curl. Enforce curl as a dependency for the libgit2 build. Fix the libgit2 clean target Build curl only on linux and osx. Add pkg-config to build dependencies in README.md. Required to detect curl in libgit2 build. --- README.md | 2 ++ contrib/windows/msys_build.sh | 1 + deps/Makefile | 8 +++++--- deps/libgit2.mk | 9 +++++++-- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index b33d164073069..121180b547eaa 100644 --- a/README.md +++ b/README.md @@ -263,6 +263,7 @@ Building Julia requires that the following software be installed: - **[patch]** — for modifying source code. - **[cmake]** — needed to build `libgit2`. - **[openssl]** — needed for HTTPS support in `libgit2` on Linux, install via `apt-get install libssl-dev` or `yum install openssl-devel`. +- **[pkg-config]** - needed to build libgit2 correctly, especially for proxy support Julia uses the following external libraries, which are automatically downloaded (or in a few cases, included in the Julia source repository) and then compiled from source the first time you run `make`: @@ -326,6 +327,7 @@ For a longer overview of Julia's dependencies, see these [slides](https://github [openssl]: https://www.openssl.org [libssh2]: https://www.libssh2.org [mbedtls]: https://tls.mbed.org/ +[pkg-config]: https://www.freedesktop.org/wiki/Software/pkg-config/ ### System Provided Libraries diff --git a/contrib/windows/msys_build.sh b/contrib/windows/msys_build.sh index 41258a21f414d..034127a9d7536 100755 --- a/contrib/windows/msys_build.sh +++ b/contrib/windows/msys_build.sh @@ -176,6 +176,7 @@ echo 'override LIBLAPACKNAME = $(LIBBLASNAME)' >> Make.user echo 'override STAGE1_DEPS = libuv' >> Make.user echo 'override STAGE2_DEPS = utf8proc' >> Make.user echo 'override STAGE3_DEPS = ' >> Make.user +echo 'override STAGE4_DEPS = ' >> Make.user if [ -n "$USEMSVC" ]; then # Openlibm doesn't build well with MSVC right now diff --git a/deps/Makefile b/deps/Makefile index 9d6dd32375950..37b4defd90884 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -171,12 +171,14 @@ ifeq ($(USE_SYSTEM_LIBSSH2), 0) STAGE2_DEPS += libssh2 endif +ifneq ($(OS), WINNT) ifeq ($(USE_SYSTEM_CURL), 0) -STAGE2_DEPS += curl +STAGE3_DEPS += curl +endif endif ifeq ($(USE_SYSTEM_LIBGIT2), 0) -STAGE3_DEPS += libgit2 +STAGE4_DEPS += libgit2 endif ifeq ($(USE_SYSTEM_MPFR), 0) @@ -215,7 +217,7 @@ endif ## Common build target prefixes -DEP_LIBS := $(STAGE1_DEPS) $(STAGE2_DEPS) $(STAGE3_DEPS) +DEP_LIBS := $(STAGE1_DEPS) $(STAGE2_DEPS) $(STAGE3_DEPS) $(STAGE4_DEPS) default: install | $(build_prefix) get: $(addprefix get-, $(DEP_LIBS)) diff --git a/deps/libgit2.mk b/deps/libgit2.mk index 113fbcea7c388..0e530132f7732 100644 --- a/deps/libgit2.mk +++ b/deps/libgit2.mk @@ -19,6 +19,8 @@ else LIBGIT2_OPTS += -DBUILD_CLAR=OFF -DDLLTOOL=`which $(CROSS_COMPILE)dlltool` LIBGIT2_OPTS += -DCMAKE_FIND_ROOT_PATH=/usr/$(XC_HOST) -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY endif +else +LIBGIT2_OPTS += -DCURL_INCLUDE_DIRS=$(build_includedir) -DCURL_LIBRARIES="-L$(build_shlibdir) -lcurl" endif ifeq ($(OS),Linux) @@ -44,6 +46,9 @@ $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-openssl-hang.patch-applied: | $(SR ifeq ($(OS),Linux) $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-require-openssl.patch-applied $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-openssl-hang.patch-applied endif +ifneq ($(OS),WINNT) +$(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(CURL_OBJ_TARGET) +endif $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/CMakeLists.txt $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-ssh.patch-applied $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-agent-nonfatal.patch-applied $(LIBSSH2_OBJ_TARGET) mkdir -p $(dir $@) cd $(dir $@) && \ @@ -73,14 +78,14 @@ ifeq ($(OS),Linux) for LIB in libssl libcrypto; do \ LIB_PATH=$$(echo "$$LIBGIT_LIBS" | grep "$$LIB"); \ echo "LIB_PATH for $$LIB: $$LIB_PATH"; \ - [ ! -z "$$LIB_PATH" ] && cp -v "$$LIB_PATH" $(build_shlibdir); \ + [ ! -z "$$LIB_PATH" ] && cp -v -f "$$LIB_PATH" $(build_shlibdir); \ done endif touch -c $@ clean-libgit2: -$(MAKE) -C $(BUILDDIR)/$(LIBGIT2_SRC_DIR) clean - -rm -f $(LIBGIT2_OBJ_TARGET) + -rm -f $(build_shlibdir)/libgit2* $(build_shlibdir)/libssl* $(build_shlibdir)/libcrypto* get-libgit2: $(LIBGIT2_SRC_FILE) configure-libgit2: $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile