Merge pull request #17391 from JuliaLang/art/pkg-lib-openssl

LibGit2: build libgit2 with libssh2 support

.travis.yml

@@ -9,7 +9,9 @@ matrix:
         apt:
           sources:
             - ubuntu-toolchain-r-test
+            - kalakris-cmake
           packages:
+            - cmake
             - bar
             - time
             - binutils
@@ -28,7 +30,9 @@ matrix:
         apt:
           sources:
             - ubuntu-toolchain-r-test
+            - kalakris-cmake
           packages:
+            - cmake
             - bar
             - time
             - g++-5
@@ -79,7 +83,7 @@ before_install:
         BUILDOPTS="-j3 USECLANG=1 LLVM_CONFIG=$(brew --prefix llvm37-julia)/bin/llvm-config-3.7.1 LLVM_SIZE=$(brew --prefix llvm37-julia)/bin/llvm-size-3.7.1";
         BUILDOPTS="$BUILDOPTS VERBOSE=1 USE_BLAS64=0 SUITESPARSE_INC=-I$(brew --prefix suite-sparse-julia)/include FORCE_ASSERTIONS=1 STAGE2_DEPS=utf8proc";
         BUILDOPTS="$BUILDOPTS LIBBLAS=-lopenblas LIBBLASNAME=libopenblas LIBLAPACK=-lopenblas LIBLAPACKNAME=libopenblas";
-        for lib in LLVM SUITESPARSE ARPACK BLAS FFTW LAPACK GMP MPFR PCRE LIBUNWIND LIBGIT2; do
+        for lib in LLVM SUITESPARSE ARPACK BLAS FFTW LAPACK GMP MPFR PCRE LIBUNWIND; do
             export BUILDOPTS="$BUILDOPTS USE_SYSTEM_$lib=1";
         done;
         export CXXFLAGS=-DUSE_ORCJIT;

LICENSE.md

@@ -50,14 +50,15 @@ Julia's standard library uses the following external libraries, which have
 their own licenses:
 
 - [ARPACK](http://www.caam.rice.edu/software/ARPACK/RiceBSD.txt#LICENSE) [BSD-3]
-- [ATLAS](http://math-atlas.sourceforge.net/faq.html#license) [BSD-3]
 - [DSFMT](http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/LICENSE.txt) [BSD-3]
 - [OPENLIBM](https://github.com/JuliaLang/openlibm/blob/master/LICENSE.md) [MIT, BSD-2, ISC]
 - [OPENSPECFUN](https://github.com/JuliaLang/openspecfun) [MIT, public domain]
 - [FADDEEVA](http://ab-initio.mit.edu/Faddeeva) [MIT]
 - [FFTW](http://fftw.org/doc/License-and-Copyright.html) [GPL2+]
 - [GMP](http://gmplib.org/manual/Copying.html#Copying) [LGPL3+ or GPL2+]
 - [LIBGIT2](https://github.com/libgit2/libgit2/blob/development/COPYING) [GPL2+ with unlimited linking exception]
+- [LIBSSH2](https://github.com/libssh2/libssh2/blob/master/COPYING) [BSD-3]
+- [MBEDTLS](https://tls.mbed.org/how-to-get) [either GPLv2 or Apache 2.0]
 - [MPFR](http://www.mpfr.org/mpfr-current/mpfr.html#Copying) [LGPL3+]
 - [OPENBLAS](https://raw.github.com/xianyi/OpenBLAS/master/LICENSE) [BSD-3]
 - [LAPACK](http://netlib.org/lapack/LICENSE.txt) [BSD-3]
@@ -86,4 +87,5 @@ Julia bundles the following external programs and libraries on some platforms:
 - [LIBEXPAT](http://expat.cvs.sourceforge.net/viewvc/expat/expat/README)
 - [OPENSSL](https://github.com/openssl/openssl/blob/master/LICENSE)
 
-On some platforms, distributions of Julia contain SSL certificate authority certificates, released under the [Mozilla Public License](https://en.wikipedia.org/wiki/Mozilla_Public_License).
+On some platforms, distributions of Julia contain SSL certificate authority certificates,
+released under the [Mozilla Public License](https://en.wikipedia.org/wiki/Mozilla_Public_License).

Make.inc

@@ -36,6 +36,8 @@ USE_SYSTEM_ARPACK:=0
 USE_SYSTEM_SUITESPARSE:=0
 USE_SYSTEM_LIBUV:=0
 USE_SYSTEM_UTF8PROC:=0
+USE_SYSTEM_MBEDTLS:=0
+USE_SYSTEM_LIBSSH2:=0
 USE_SYSTEM_LIBGIT2:=0
 USE_SYSTEM_PATCHELF:=0
 

Makefile

@@ -271,6 +271,12 @@ endif
 ifeq ($(USE_SYSTEM_MPFR),0)
 JL_PRIVATE_LIBS += mpfr
 endif
+ifeq ($(USE_SYSTEM_MBEDTLS),0)
+JL_PRIVATE_LIBS += mbedtls mbedcrypto mbedx509
+endif
+ifeq ($(USE_SYSTEM_LIBSSH2),0)
+JL_PRIVATE_LIBS += ssh2
+endif
 ifeq ($(USE_SYSTEM_LIBGIT2),0)
 JL_PRIVATE_LIBS += git2
 endif

README.md

@@ -282,7 +282,9 @@ Julia uses the following external libraries, which are automatically downloaded
 - **[PCRE]** (>= 10.00)      — Perl-compatible regular expressions library.
 - **[GMP]** (>= 5.0)         — GNU multiple precision arithmetic library, needed for `BigInt` support.
 - **[MPFR]** (>= 3.0)        — GNU multiple precision floating point library, needed for arbitrary precision floating point (`BigFloat`) support.
-- **[libgit2]** (>= 0.21)    — Git linkable library, used by Julia's package manager
+- **[libgit2]** (>= 0.24)    — Git linkable library, used by Julia's package manager
+- **[libssh2]** (>= 1.7)     — library for SSH transport, used by libgit2 for packages with SSH remotes
+- **[mbedtls]** (>= 2.2)     — library used for cryptography and transport layer security, used by libssh2
 - **[utf8proc]** (>= 2.0)    — a library for processing UTF-8 encoded Unicode strings
 - **[libosxunwind]**         — clone of [libunwind], a library that determines the call-chain of a program
 
@@ -321,6 +323,8 @@ For a longer overview of Julia's dependencies, see these [slides](https://github
 [libosxunwind]: https://github.com/JuliaLang/libosxunwind
 [libunwind]:    http://www.nongnu.org/libunwind
 [openssl]:      https://www.openssl.org
+[libssh2]:      https://www.libssh2.org
+[mbedtls]:      https://tls.mbed.org/
 
 <a name="System-Provided-Libraries">
 ### System Provided Libraries

base/libgit2/libgit2.jl

@@ -520,16 +520,19 @@ function __init__()
         ccall((:git_libgit2_shutdown, :libgit2), Cint, ())
     end
 
-    # Look for OpenSSL env variable for CA bundle
-    cert_loc = if "SSL_CERT_DIR" in keys(ENV)
-        ENV["SSL_CERT_DIR"]
-    elseif "SSL_CERT_FILE" in keys(ENV)
-        ENV["SSL_CERT_FILE"]
-    else
-        # If we have a bundled ca cert file, point libgit2 at that so SSL connections work.
-        abspath(ccall(:jl_get_julia_home, Any, ()),Base.DATAROOTDIR,"julia","cert.pem")
+    # Look for OpenSSL env variable for CA bundle (linux only)
+    # windows and macOS use the OS native security backends
+    @static if is_linux()
+        cert_loc = if "SSL_CERT_DIR" in keys(ENV)
+            ENV["SSL_CERT_DIR"]
+        elseif "SSL_CERT_FILE" in keys(ENV)
+            ENV["SSL_CERT_FILE"]
+        else
+            # If we have a bundled ca cert file, point libgit2 at that so SSL connections work.
+            abspath(ccall(:jl_get_julia_home, Any, ()),Base.DATAROOTDIR,"julia","cert.pem")
+        end
+        set_ssl_cert_locations(cert_loc)
     end
-    set_ssl_cert_locations(cert_loc)
 end
 
 

contrib/windows/msys_build.sh

@@ -173,9 +173,9 @@ echo 'override LIBLAPACKNAME = $(LIBBLASNAME)' >> Make.user
 # libuv since its static lib is no longer included in the binaries
 # openlibm since we need it as a static library to work properly
 # utf8proc since its headers are not in the binary download
-echo 'override STAGE1_DEPS = libuv' >> Make.user
-echo 'override STAGE2_DEPS = utf8proc' >> Make.user
-echo 'override STAGE3_DEPS = ' >> Make.user
+echo 'override STAGE1_DEPS = libuv mbedtls' >> Make.user
+echo 'override STAGE2_DEPS = utf8proc libssh2' >> Make.user
+echo 'override STAGE3_DEPS = libgit2' >> Make.user
 
 if [ -n "$USEMSVC" ]; then
   # Openlibm doesn't build well with MSVC right now

deps/Makefile

@@ -20,9 +20,9 @@ include $(SRCDIR)/llvm-ver.make
 # autoconf configure-driven scripts: llvm pcre arpack fftw unwind gmp mpfr patchelf libuv
 # custom Makefile rules: openlibm dsfmt suitesparse-wrapper suitesparse lapack openblas utf8proc objconv osxunwind
 # entirely custom: virtualenv
-# CMake libs: libgit2
+# CMake libs: libgit2 libssh2 mbedtls
 #
-# downloaded from git: llvm-svn, libuv, libopenlibm, utf8proc, openspecfun, libgit2
+# downloaded from git: llvm-svn, libuv, libopenlibm, utf8proc, openspecfun, libgit2, libssh2
 #
 # there are rules in this file with the . replaced by a %
 # this is some magic Makefile trick that tells make
@@ -46,7 +46,7 @@ CONFIGURE_COMMON += F77="$(FC)" CC="$(CC) $(DEPS_CFLAGS)" CXX="$(CXX) $(DEPS_CXX
 CMAKE_CC_ARG := $(CC_ARG) $(DEPS_CFLAGS)
 CMAKE_CXX_ARG := $(CXX_ARG) $(DEPS_CXXFLAGS)
 
-CMAKE_COMMON := -DCMAKE_INSTALL_PREFIX:PATH=$(build_prefix)
+CMAKE_COMMON := -DCMAKE_INSTALL_PREFIX:PATH=$(build_prefix) -DCMAKE_PREFIX_PATH=$(build_prefix)
 ifneq ($(VERBOSE), 0)
 CMAKE_COMMON += -DCMAKE_VERBOSE_MAKEFILE=ON
 endif
@@ -162,8 +162,16 @@ ifeq ($(USE_SYSTEM_GMP), 0)
 STAGE1_DEPS += gmp
 endif
 
+ifeq ($(USE_SYSTEM_MBEDTLS), 0)
+STAGE1_DEPS += mbedtls
+endif
+
+ifeq ($(USE_SYSTEM_LIBSSH2), 0)
+STAGE2_DEPS += libssh2
+endif
+
 ifeq ($(USE_SYSTEM_LIBGIT2), 0)
-STAGE1_DEPS += libgit2
+STAGE3_DEPS += libgit2
 endif
 
 ifeq ($(USE_SYSTEM_MPFR), 0)
@@ -213,7 +221,7 @@ install: $(addprefix install-, $(DEP_LIBS))
 cleanall: $(addprefix clean-, $(DEP_LIBS))
 distcleanall: $(addprefix distclean-, $(DEP_LIBS))
 	rm -rf $(build_prefix)
-getall: get-llvm get-libuv get-pcre get-openlibm get-openspecfun get-dsfmt get-openblas get-lapack get-fftw get-suitesparse get-arpack get-unwind get-osxunwind get-gmp get-mpfr get-patchelf get-utf8proc get-virtualenv get-objconv get-libgit2
+getall: get-llvm get-libuv get-pcre get-openlibm get-openspecfun get-dsfmt get-openblas get-lapack get-fftw get-suitesparse get-arpack get-unwind get-osxunwind get-gmp get-mpfr get-patchelf get-utf8proc get-virtualenv get-objconv get-mbedtls get-libssh2 get-libgit2
 
 ## PATHS ##
 # sort is used to remove potential duplicates
@@ -260,6 +268,8 @@ include $(SRCDIR)/unwind.mk
 include $(SRCDIR)/gmp.mk
 include $(SRCDIR)/mpfr.mk
 include $(SRCDIR)/patchelf.mk
+include $(SRCDIR)/mbedtls.mk
+include $(SRCDIR)/libssh2.mk
 include $(SRCDIR)/libgit2.mk
 include $(SRCDIR)/virtualenv.mk
 

deps/Versions.make

@@ -12,3 +12,4 @@ GMP_VER = 6.1.0
 MPFR_VER = 3.1.4
 PATCHELF_VER = 0.9
 VIRTUALENV_VER = 15.0.0
+MBEDTLS_VER = 2.2.1

deps/checksums/libgit2-211e117a0590583a720c53172406f34186c543bd.tar.gz/md5

@@ -0,0 +1 @@
+eb46efa15014aaec7e574dfdc5921a36

deps/checksums/libgit2-211e117a0590583a720c53172406f34186c543bd.tar.gz/sha512

@@ -0,0 +1 @@
+20aa4ca35e78873961ad45734db8af36fe600d523a8ef8c77dba66944c7e33efbe239c49f3258e81cf6a7f7b6afa3b867ad62fc1fb507377ad8e9fa2bb629472

deps/checksums/libssh2-6d553a7bb94966466c4db0abf9a26660e786a275.tar.gz/md5

@@ -0,0 +1 @@
+25c40973c9cdcd7919c0b74ca0d06dd1

deps/checksums/libssh2-6d553a7bb94966466c4db0abf9a26660e786a275.tar.gz/sha512

@@ -0,0 +1 @@
+eea5e3638395cb5711f97b36adfa05a9c1cb24cf3b5639cb0a20d74855da025c2db2ff3eeb6bf37d49977e2d9fcf2b929e5c5d874c65754c280defaacb9f3eb8

deps/checksums/mbedtls-2.2.1-apache.tgz/md5

@@ -0,0 +1 @@
+77751c0e370ed2ab01934e4c5e1d380f

deps/checksums/mbedtls-2.2.1-apache.tgz/sha512

@@ -0,0 +1 @@
+6a74abc4ea225eb6bcf20894bb1a6faa82dbaff11129c41849151e2654570609efeee70d0644ce63c4d2c11e6142b2db262b88f3a22fdceff0a215a64a5d6eb0

deps/checksums/mbedtls-2.2.1-gpl.tgz/md5

@@ -0,0 +1 @@
+b6909d94600fc5f644e9bca52e96fb27

deps/checksums/mbedtls-2.2.1-gpl.tgz/sha512

@@ -0,0 +1 @@
+03c32b46369d35e61acfeac83f417d09307d2f1e4358f9ea6af275d9a9a2319ca7a244733b5950a48d59f6305e4adcad6a66e589b42ee7417d1d4362c2595a4f

deps/libgit2.mk

@@ -7,9 +7,9 @@ $(eval $(call git-external,libgit2,LIBGIT2,CMakeLists.txt,build/libgit2.$(SHLIB_
 LIBGIT2_OBJ_SOURCE := $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/libgit2.$(SHLIB_EXT)
 LIBGIT2_OBJ_TARGET := $(build_shlibdir)/libgit2.$(SHLIB_EXT)
 
-LIBGIT2_OPTS := $(CMAKE_COMMON) -DTHREADSAFE=ON
+LIBGIT2_OPTS := $(CMAKE_COMMON) -DCMAKE_BUILD_TYPE=Release -DTHREADSAFE=ON
 ifeq ($(OS),WINNT)
-LIBGIT2_OPTS += -DWIN32=ON -DMINGW=ON -DUSE_SSH=OFF -DCMAKE_BUILD_TYPE=RelWithDebInfo
+LIBGIT2_OPTS += -DWIN32=ON -DMINGW=ON
 ifneq ($(ARCH),x86_64)
 LIBGIT2_OPTS += -DCMAKE_C_FLAGS="-mincoming-stack-boundary=2"
 endif
@@ -20,26 +20,35 @@ LIBGIT2_OPTS += -DBUILD_CLAR=OFF -DDLLTOOL=`which $(CROSS_COMPILE)dlltool`
 LIBGIT2_OPTS += -DCMAKE_FIND_ROOT_PATH=/usr/$(XC_HOST) -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY
 endif
 else
-LIBGIT2_OPTS += -DCMAKE_BUILD_TYPE=Release
+LIBGIT2_OPTS += -DCMAKE_INSTALL_RPATH=$(build_prefix) -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=TRUE
 endif
 
-$(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/CMakeLists.txt
+$(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/CMakeLists.txt $(LIBSSH2_OBJ_TARGET)
+ifeq ($(OS),Linux)
 	-cd $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR) && patch -p1 -f < $(SRCDIR)/patches/libgit2-require-openssl.patch
+endif
+	-cd $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR) && patch -p0 -f < $(SRCDIR)/patches/libgit2-ssh.patch
 	mkdir -p $(dir $@)
 	cd $(dir $@) && \
 	$(CMAKE) $(dir $<) $(LIBGIT2_OPTS)
 	touch -c $@
+
 $(LIBGIT2_OBJ_SOURCE): $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile
 	$(MAKE) -C $(dir $<)
 	touch -c $@
+
 $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/checked: $(LIBGIT2_OBJ_SOURCE)
 ifeq ($(OS),$(BUILD_OS))
 	$(MAKE) -C $(dir $@) test
 endif
 	echo 1 > $@
+
 $(LIBGIT2_OBJ_TARGET): $(LIBGIT2_OBJ_SOURCE) | $(build_shlibdir)
+ifeq ($(BUILD_OS),WINNT)
 	cp $< $@
-	@#$$(call make-install,$(LIBGIT2_SRC_DIR),) # currently don't need the full install
+else
+	$(call make-install,$(LIBGIT2_SRC_DIR),)
+endif
 ifeq ($(OS),Linux)
 	@# If we're on linux, copy over libssl and libcrypto for libgit2
 	-LIBGIT_LIBS=$$(ldd "$@" | tail -n +2 | awk '{print $$(NF-1)}'); \
@@ -52,7 +61,7 @@ endif
 	touch -c $@
 
 clean-libgit2:
-	-rm -rf $(BUILDDIR)/$(LIBGIT2_SRC_DIR)
+	-$(MAKE) -C $(BUILDDIR)/$(LIBGIT2_SRC_DIR) clean
 	-rm -f $(LIBGIT2_OBJ_TARGET)
 
 get-libgit2: $(LIBGIT2_SRC_FILE)

deps/libgit2.version

@@ -1,2 +1,2 @@
-LIBGIT2_BRANCH=v0.24.0
-LIBGIT2_SHA1=785d8c48ea8725691da3c50e7dae8751523d4c30
+LIBGIT2_BRANCH=v0.24.1
+LIBGIT2_SHA1=211e117a0590583a720c53172406f34186c543bd

deps/libssh2.mk

@@ -0,0 +1,56 @@
+## libssh2
+
+LIBSSH2_GIT_URL := git://github.com/libssh2/libssh2.git
+LIBSSH2_TAR_URL = https://api.github.com/repos/libssh2/libssh2/tarball/$1
+$(eval $(call git-external,libssh2,LIBSSH2,CMakeLists.txt,build/libssh2.$(SHLIB_EXT),$(SRCDIR)/srccache))
+
+LIBSSH2_OBJ_SOURCE := $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/src/libssh2.$(SHLIB_EXT)
+LIBSSH2_OBJ_TARGET := $(build_shlibdir)/libssh2.$(SHLIB_EXT)
+
+LIBSSH2_OPTS := $(CMAKE_COMMON) -DBUILD_SHARED_LIBS=ON -DBUILD_EXAMPLES=OFF \
+				-DCMAKE_INSTALL_RPATH=$(build_prefix) -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=TRUE \
+				-DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_LIBDIR=lib
+
+ifeq ($(OS),WINNT)
+LIBSSH2_OPTS += -DCRYPTO_BACKEND=WinCNG -DENABLE_ZLIB_COMPRESSION=OFF
+ifeq ($(BUILD_OS),WINNT)
+LIBSSH2_OPTS += -G"MSYS Makefiles"
+endif
+else
+LIBSSH2_OPTS += -DCRYPTO_BACKEND=mbedTLS -DENABLE_ZLIB_COMPRESSION=ON
+endif
+
+$(BUILDDIR)/$(LIBSSH2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/CMakeLists.txt $(MBEDTLS_OBJ_TARGET)
+	-cd $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR) && patch -p1 -f < $(SRCDIR)/patches/libssh2-mbedtls.patch
+	mkdir -p $(dir $@)
+	cd $(dir $@) && \
+	$(CMAKE) $(dir $<) $(LIBSSH2_OPTS)
+	touch -c $@
+
+$(LIBSSH2_OBJ_SOURCE): $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/Makefile
+	$(MAKE) -C $(dir $<) libssh2
+	touch -c $@
+
+$(BUILDDIR)/$(LIBSSH2_SRC_DIR)/checked: $(LIBSSH2_OBJ_SOURCE)
+ifeq ($(OS),$(BUILD_OS))
+	$(MAKE) -C $(dir $@) test
+endif
+	echo 1 > $@
+
+$(LIBSSH2_OBJ_TARGET): $(LIBSSH2_OBJ_SOURCE) | $(build_shlibdir)
+ifeq ($(BUILD_OS),WINNT)
+	$(MAKE) -C $(BUILDDIR)/$(LIBSSH2_SRC_DIR) install
+else
+	$(call make-install,$(LIBSSH2_SRC_DIR),)
+endif
+	touch -c $(LIBSSH2_OBJ_TARGET)
+
+clean-libssh2:
+	-$(MAKE) -C $(BUILDDIR)/$(LIBSSH2_SRC_DIR) clean
+	-rm -f $(LIBSSH2_OBJ_TARGET)
+
+get-libssh2: $(LIBSSH2_SRC_FILE)
+configure-libssh2: $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/Makefile
+compile-libssh2: $(LIBSSH2_OBJ_SOURCE)
+check-libssh2: $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/checked
+install-libssh2: $(LIBSSH2_OBJ_TARGET)

deps/libssh2.version

@@ -0,0 +1,2 @@
+LIBSSH2_BRANCH=libssh2-1.7.0
+LIBSSH2_SHA1=6d553a7bb94966466c4db0abf9a26660e786a275