Merge pull request #17783 from JuliaLang/vs/curl

Source build of curl to enable proxy support for Pkg

LICENSE.md

@@ -57,6 +57,7 @@ their own licenses:
 - [FFTW](http://fftw.org/doc/License-and-Copyright.html) [GPL2+]
 - [GMP](http://gmplib.org/manual/Copying.html#Copying) [LGPL3+ or GPL2+]
 - [LIBGIT2](https://github.com/libgit2/libgit2/blob/development/COPYING) [GPL2+ with unlimited linking exception]
+- [CURL](https://curl.haxx.se/docs/copyright.html) [MIT/X derivative]
 - [LIBSSH2](https://github.com/libssh2/libssh2/blob/master/COPYING) [BSD-3]
 - [MBEDTLS](https://tls.mbed.org/how-to-get) [either GPLv2 or Apache 2.0]
 - [MPFR](http://www.mpfr.org/mpfr-current/mpfr.html#Copying) [LGPL3+]

Make.inc

@@ -38,6 +38,7 @@ USE_SYSTEM_LIBUV:=0
 USE_SYSTEM_UTF8PROC:=0
 USE_SYSTEM_MBEDTLS:=0
 USE_SYSTEM_LIBSSH2:=0
+USE_SYSTEM_CURL:=0
 USE_SYSTEM_LIBGIT2:=0
 USE_SYSTEM_PATCHELF:=0
 

Makefile

@@ -277,6 +277,9 @@ endif
 ifeq ($(USE_SYSTEM_LIBSSH2),0)
 JL_PRIVATE_LIBS += ssh2
 endif
+ifeq ($(USE_SYSTEM_CURL),0)
+JL_PRIVATE_LIBS += curl
+endif
 ifeq ($(USE_SYSTEM_LIBGIT2),0)
 JL_PRIVATE_LIBS += git2
 endif

README.md

@@ -263,6 +263,7 @@ Building Julia requires that the following software be installed:
 - **[patch]**                   — for modifying source code.
 - **[cmake]**                   — needed to build `libgit2`.
 - **[openssl]**                 — needed for HTTPS support in `libgit2` on Linux, install via `apt-get install libssl-dev` or `yum install openssl-devel`.
+- **[pkg-config]**              - needed to build libgit2 correctly, especially for proxy support
 
 Julia uses the following external libraries, which are automatically downloaded (or in a few cases, included in the Julia source repository) and then compiled from source the first time you run `make`:
 
@@ -283,6 +284,7 @@ Julia uses the following external libraries, which are automatically downloaded
 - **[GMP]** (>= 5.0)         — GNU multiple precision arithmetic library, needed for `BigInt` support.
 - **[MPFR]** (>= 3.0)        — GNU multiple precision floating point library, needed for arbitrary precision floating point (`BigFloat`) support.
 - **[libgit2]** (>= 0.23)    — Git linkable library, used by Julia's package manager
+- **[curl]** (>= 7.50)       — libcurl provides download and proxy support for Julia's package manager
 - **[libssh2]** (>= 1.7)     — library for SSH transport, used by libgit2 for packages with SSH remotes
 - **[mbedtls]** (>= 2.2)     — library used for cryptography and transport layer security, used by libssh2
 - **[utf8proc]** (>= 2.0)    — a library for processing UTF-8 encoded Unicode strings
@@ -325,6 +327,7 @@ For a longer overview of Julia's dependencies, see these [slides](https://github
 [openssl]:      https://www.openssl.org
 [libssh2]:      https://www.libssh2.org
 [mbedtls]:      https://tls.mbed.org/
+[pkg-config]:   https://www.freedesktop.org/wiki/Software/pkg-config/
 
 <a name="System-Provided-Libraries">
 ### System Provided Libraries

contrib/windows/msys_build.sh

@@ -176,6 +176,7 @@ echo 'override LIBLAPACKNAME = $(LIBBLASNAME)' >> Make.user
 echo 'override STAGE1_DEPS = libuv' >> Make.user
 echo 'override STAGE2_DEPS = utf8proc' >> Make.user
 echo 'override STAGE3_DEPS = ' >> Make.user
+echo 'override STAGE4_DEPS = ' >> Make.user
 
 if [ -n "$USEMSVC" ]; then
   # Openlibm doesn't build well with MSVC right now

deps/Makefile

@@ -17,7 +17,7 @@ include $(SRCDIR)/llvm-ver.make
 # additionally all targets should be listed in the getall target for easier off-line compilation
 # if you are adding a new target, it can help to copy an similar, existing target
 #
-# autoconf configure-driven scripts: llvm pcre arpack fftw unwind gmp mpfr patchelf libuv
+# autoconf configure-driven scripts: llvm pcre arpack fftw unwind gmp mpfr patchelf libuv curl
 # custom Makefile rules: openlibm dsfmt suitesparse-wrapper suitesparse lapack openblas utf8proc objconv osxunwind
 # entirely custom: virtualenv
 # CMake libs: libgit2 libssh2 mbedtls
@@ -171,8 +171,14 @@ ifeq ($(USE_SYSTEM_LIBSSH2), 0)
 STAGE2_DEPS += libssh2
 endif
 
+ifneq ($(OS), WINNT)
+ifeq ($(USE_SYSTEM_CURL), 0)
+STAGE3_DEPS += curl
+endif
+endif
+
 ifeq ($(USE_SYSTEM_LIBGIT2), 0)
-STAGE3_DEPS += libgit2
+STAGE4_DEPS += libgit2
 endif
 
 ifeq ($(USE_SYSTEM_MPFR), 0)
@@ -211,7 +217,7 @@ endif
 
 ## Common build target prefixes
 
-DEP_LIBS := $(STAGE1_DEPS) $(STAGE2_DEPS) $(STAGE3_DEPS)
+DEP_LIBS := $(STAGE1_DEPS) $(STAGE2_DEPS) $(STAGE3_DEPS) $(STAGE4_DEPS)
 
 default: install | $(build_prefix)
 get: $(addprefix get-, $(DEP_LIBS))
@@ -222,7 +228,7 @@ install: $(addprefix install-, $(DEP_LIBS))
 cleanall: $(addprefix clean-, $(DEP_LIBS))
 distcleanall: $(addprefix distclean-, $(DEP_LIBS))
 	rm -rf $(build_prefix)
-getall: get-llvm get-libuv get-pcre get-openlibm get-openspecfun get-dsfmt get-openblas get-lapack get-fftw get-suitesparse get-arpack get-unwind get-osxunwind get-gmp get-mpfr get-patchelf get-utf8proc get-virtualenv get-objconv get-mbedtls get-libssh2 get-libgit2
+getall: get-llvm get-libuv get-pcre get-openlibm get-openspecfun get-dsfmt get-openblas get-lapack get-fftw get-suitesparse get-arpack get-unwind get-osxunwind get-gmp get-mpfr get-patchelf get-utf8proc get-virtualenv get-objconv get-mbedtls get-libssh2 get-curl get-libgit2
 
 ## PATHS ##
 # sort is used to remove potential duplicates
@@ -271,6 +277,7 @@ include $(SRCDIR)/mpfr.mk
 include $(SRCDIR)/patchelf.mk
 include $(SRCDIR)/mbedtls.mk
 include $(SRCDIR)/libssh2.mk
+include $(SRCDIR)/curl.mk
 include $(SRCDIR)/libgit2.mk
 include $(SRCDIR)/virtualenv.mk
 

deps/Versions.make

@@ -13,3 +13,4 @@ MPFR_VER = 3.1.4
 PATCHELF_VER = 0.9
 VIRTUALENV_VER = 15.0.0
 MBEDTLS_VER = 2.3.0
+CURL_VER = 7.50.1

deps/checksums/curl-7.50.1.tar.bz2/md5

@@ -0,0 +1 @@
+015f6a0217ca6f2c5442ca406476920b

deps/checksums/curl-7.50.1.tar.bz2/sha512

@@ -0,0 +1 @@
+94acd91fcf8ff2605e1ba2d086c2c366257b61eaf516b9ea44e574e315feb5b30f6e47d89051f259e026ef5dd9edde5f7b15a6af9ee6a38f641da354e1e677b1

deps/curl.mk

@@ -0,0 +1,46 @@
+## CURL ##
+
+CURL_SRC_TARGET := $(BUILDDIR)/curl-$(CURL_VER)/.libs/libcurl.$(SHLIB_EXT)
+CURL_OBJ_TARGET := $(build_shlibdir)/libcurl.$(SHLIB_EXT)
+
+ifneq ($(OS),WINNT)
+CURL_LDFLAGS := $(RPATH_ESCAPED_ORIGIN)
+endif
+
+$(SRCDIR)/srccache/curl-$(CURL_VER).tar.bz2: | $(SRCDIR)/srccache
+	$(JLDOWNLOAD) $@ https://curl.haxx.se/download/curl-$(CURL_VER).tar.bz2
+$(SRCDIR)/srccache/curl-$(CURL_VER)/configure: $(SRCDIR)/srccache/curl-$(CURL_VER).tar.bz2 $(MBEDTLS_OBJ_TARGET) $(LIBSSH2_OBJ_TARGET)
+	$(JLCHECKSUM) $<
+	cd $(dir $<) && $(TAR) jxf $(notdir $<)
+	touch -c $@
+$(BUILDDIR)/curl-$(CURL_VER)/config.status: $(SRCDIR)/srccache/curl-$(CURL_VER)/configure
+	mkdir -p $(dir $@)
+	cd $(dir $@) && \
+	$< $(CONFIGURE_COMMON) --includedir=$(build_includedir) --with-mbedtls=$(build_shlibdir)/.. CFLAGS="$(CFLAGS) $(CURL_CFLAGS)" LDFLAGS="$(LDFLAGS) $(CURL_LDFLAGS)"
+	touch -c $@
+$(CURL_SRC_TARGET): $(BUILDDIR)/curl-$(CURL_VER)/config.status
+	$(MAKE) -C $(dir $<) $(LIBTOOL_CCLD)
+	touch -c $@
+$(BUILDDIR)/curl-$(CURL_VER)/checked: $(CURL_SRC_TARGET)
+ifeq ($(OS),$(BUILD_OS))
+ifneq ($(OS),WINNT)
+	$(MAKE) -C $(dir $@) check -j1
+endif
+endif
+	echo 1 > $@
+$(CURL_OBJ_TARGET): $(CURL_SRC_TARGET)
+	$(call make-install,curl-$(CURL_VER),$(LIBTOOL_CCLD))
+	$(INSTALL_NAME_CMD)libcurl.$(SHLIB_EXT) $@
+	touch -c $@
+
+clean-curl:
+	-$(MAKE) -C $(BUILDDIR)/curl-$(CURL_VER) clean
+	-rm -f $(build_shlibdir)/libcurl*
+distclean-curl:
+	-rm -rf $(SRCDIR)/srccache/curl-$(CURL_VER).tar.bz2 $(SRCDIR)/srccache/curl-$(CURL_VER) $(BUILDDIR)/curl-$(CURL_VER)
+
+get-curl: $(SRCDIR)/srccache/curl-$(CURL_VER).tar.bz2
+configure-curl: $(BUILDDIR)/curl-$(CURL_VER)/config.status
+compile-curl: $(CURL_SRC_TARGET)
+check-curl: $(BUILDDIR)/curl-$(CURL_VER)/checked
+install-curl: $(CURL_OBJ_TARGET)

deps/libgit2.mk

@@ -19,6 +19,8 @@ else
 LIBGIT2_OPTS += -DBUILD_CLAR=OFF -DDLLTOOL=`which $(CROSS_COMPILE)dlltool`
 LIBGIT2_OPTS += -DCMAKE_FIND_ROOT_PATH=/usr/$(XC_HOST) -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY
 endif
+else
+LIBGIT2_OPTS += -DCURL_INCLUDE_DIRS=$(build_includedir) -DCURL_LIBRARIES="-L$(build_shlibdir) -lcurl"
 endif
 
 ifeq ($(OS),Linux)
@@ -44,6 +46,9 @@ $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-openssl-hang.patch-applied: | $(SR
 ifeq ($(OS),Linux)
 $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-require-openssl.patch-applied $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-openssl-hang.patch-applied
 endif
+ifneq ($(OS),WINNT)
+$(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(CURL_OBJ_TARGET)
+endif
 $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/CMakeLists.txt $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-ssh.patch-applied $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-agent-nonfatal.patch-applied $(LIBSSH2_OBJ_TARGET)
 	mkdir -p $(dir $@)
 	cd $(dir $@) && \
@@ -73,14 +78,14 @@ ifeq ($(OS),Linux)
 	for LIB in libssl libcrypto; do \
 		LIB_PATH=$$(echo "$$LIBGIT_LIBS" | grep "$$LIB"); \
 		echo "LIB_PATH for $$LIB: $$LIB_PATH"; \
-		[ ! -z "$$LIB_PATH" ] && cp -v "$$LIB_PATH" $(build_shlibdir); \
+		[ ! -z "$$LIB_PATH" ] && cp -v -f "$$LIB_PATH" $(build_shlibdir); \
 	done
 endif
 	touch -c $@
 
 clean-libgit2:
 	-$(MAKE) -C $(BUILDDIR)/$(LIBGIT2_SRC_DIR) clean
-	-rm -f $(LIBGIT2_OBJ_TARGET)
+	-rm -f $(build_shlibdir)/libgit2* $(build_shlibdir)/libssl* $(build_shlibdir)/libcrypto*
 
 get-libgit2: $(LIBGIT2_SRC_FILE)
 configure-libgit2: $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile

deps/mbedtls.mk

@@ -39,7 +39,11 @@ $(SRCDIR)/srccache/$(MBEDTLS_SRC)/mbedtls-config.patch-applied: | $(SRCDIR)/srcc
 	cd $(SRCDIR)/srccache/$(MBEDTLS_SRC) && patch -p0 -f < $(SRCDIR)/patches/mbedtls-config.patch
 	echo 1 > $@
 
-$(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/Makefile: $(SRCDIR)/srccache/$(MBEDTLS_SRC)/CMakeLists.txt $(SRCDIR)/srccache/$(MBEDTLS_SRC)/mbedtls-config.patch-applied
+$(SRCDIR)/srccache/$(MBEDTLS_SRC)/mbedtls-ssl.h.patch-applied: | $(SRCDIR)/srccache/$(MBEDTLS_SRC)/CMakeLists.txt
+	cd $(SRCDIR)/srccache/$(MBEDTLS_SRC)/include/mbedtls && patch -p0 -f < $(SRCDIR)/patches/mbedtls-ssl.h.patch
+	echo 1 > $@
+
+$(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/Makefile: $(SRCDIR)/srccache/$(MBEDTLS_SRC)/CMakeLists.txt $(SRCDIR)/srccache/$(MBEDTLS_SRC)/mbedtls-config.patch-applied $(SRCDIR)/srccache/$(MBEDTLS_SRC)/mbedtls-ssl.h.patch-applied
 	mkdir -p $(dir $@)
 	cd $(dir $@) && \
 	$(CMAKE) $(dir $<) $(MBEDTLS_OPTS)
@@ -71,7 +75,7 @@ endif
 
 clean-mbedtls:
 	-$(MAKE) -C $(BUILDDIR)/mbedtls-$(MBEDTLS_VER) clean
-	-rm -f $(MBEDTLS_OBJ_TARGET)
+	-rm -f $(MBEDTLS_OBJ_TARGET) $(build_shlibdir)/libmbed*
 
 distclean-mbedtls:
 	-rm -rf $(SRCDIR)/srccache/$(MBEDTLS_SRC).tgz \

deps/patches/mbedtls-ssl.h.patch

@@ -0,0 +1,12 @@
+--- ssl.h.old	2016-06-28 18:12:06.000000000 +0530
++++ ssl.h	2016-08-03 18:51:34.000000000 +0530
+@@ -54,7 +54,8 @@
+ #endif
+
+ #if defined(MBEDTLS_HAVE_TIME)
+-#include <time.h>
++//#include <time.h>
++#include "platform.h"
+ #endif
+
+ /*

deps/pcre.mk

@@ -36,7 +36,7 @@ $(PCRE_OBJ_TARGET): $(PCRE_SRC_TARGET)
 	touch -c $@
 
 clean-pcre:
-	-$(MAKE) -C pcre2-$(PCRE_VER) clean
+	-$(MAKE) -C $(BUILDDIR)/pcre2-$(PCRE_VER) clean
 	-rm -f $(build_shlibdir)/libpcre*
 distclean-pcre:
 	-rm -rf $(SRCDIR)/srccache/pcre2-$(PCRE_VER).tar.bz2 $(SRCDIR)/srccache/pcre2-$(PCRE_VER) $(BUILDDIR)/pcre2-$(PCRE_VER)