From 01fd54cf5a7b859d63d6d82ac60fd813f729da9f Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 4 Dec 2017 20:43:23 +0100 Subject: [PATCH 01/16] contrib/buildsystems: error out on unknown option One time too many did this developer call the `generate` script passing a `--make-out=` option that was happily ignored (because there should be a space, not an equal sign, between `--make-out` and the path). And one time too many, this script not only ignored it but did not even complain. Let's fix that. Signed-off-by: Johannes Schindelin --- contrib/buildsystems/engine.pl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/buildsystems/engine.pl b/contrib/buildsystems/engine.pl index 6cfaedb029364d..35fd05633a5626 100755 --- a/contrib/buildsystems/engine.pl +++ b/contrib/buildsystems/engine.pl @@ -57,6 +57,8 @@ sub showUsage open(F, "<$infile") || die "Couldn't open file $infile"; @makedry = ; close(F); + } else { + die "Unknown option: " . $arg; } } From 9494fd0afa80476ef2a376c02213b2eb36afb2eb Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 4 Dec 2017 20:47:24 +0100 Subject: [PATCH 02/16] contrib/buildsystems: redirect stderr into the correct directory The script assumes that we're in the top-level directory of the checkout. That does not need to be true. Signed-off-by: Johannes Schindelin --- contrib/buildsystems/engine.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/buildsystems/engine.pl b/contrib/buildsystems/engine.pl index 35fd05633a5626..9f4e7a2ccb9352 100755 --- a/contrib/buildsystems/engine.pl +++ b/contrib/buildsystems/engine.pl @@ -82,7 +82,7 @@ sub showUsage # Capture the make dry stderr to file for review (will be empty for a release build). my $ErrsFile = "msvc-build-makedryerrors.txt"; -@makedry = `cd $git_dir && make -n MSVC=1 V=1 2>$ErrsFile` if !@makedry; +@makedry = `make -C $git_dir -n MSVC=1 V=1 2>$ErrsFile` if !@makedry; # test for an empty Errors file and remove it unlink $ErrsFile if -f -z $ErrsFile; From e760a820d09d12728c97b12b90300ddcc091af2e Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 28 Nov 2017 18:02:51 +0100 Subject: [PATCH 03/16] Mark .bat files as requiring CR/LF endings Signed-off-by: Johannes Schindelin --- .gitattributes | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitattributes b/.gitattributes index 8ce9c6b8888fe6..9682d55461eb7a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,6 +3,7 @@ *.sh whitespace=indent,trail,space eol=lf *.perl eol=lf *.pm eol=lf +*.bat eol=crlf /Documentation/git-*.txt eol=lf /command-list.txt eol=lf /GIT-VERSION-GEN eol=lf From 07784d4b1eb209d0df6cd44fabada5169a36bf2e Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 19 Sep 2017 12:56:21 -0400 Subject: [PATCH 04/16] vcpkg: get MSVC dependencies with vcpkg rather than nuget Dependencies such as cURL and OpenSSL are necessary to build and run Git. Previously, we obtained those dependencies by fetching NuGet packages. However, it is notoriously hard to keep NuGet packages of C/C++ libraries up-to-date, as the toolsets for different Visual Studio versions are different, and the NuGet packages would have to ship them all. That is the reason why the NuGet packages we use are quite old, and even insecure in the case of cURL and OpenSSL (the versions contain known security flaws that have been addressed by later versions for which no NuGet packages are available). The better way to handle this situation is to use the vcpkg system: https://github.com/Microsoft/vcpkg The idea is that a single Git repository contains enough supporting files to build up-to-date versions of a large number of Open Source libraries on demand, including cURL and OpenSSL. We integrate this system via four new .bat files to 1) initialize the vcpkg system, 2) build the packages, 4) set up Git's Makefile system to find the build artifacts, and 3) copy the artifacts into the top-level directory Signed-off-by: Jeff Hostetler Signed-off-by: Johannes Schindelin --- .gitignore | 1 - Makefile | 10 +- compat/vcbuild/.gitignore | 4 +- compat/vcbuild/README | 51 +++++ compat/vcbuild/find_vs_env.bat | 334 ++++++++++++++--------------- compat/vcbuild/scripts/clink.pl | 15 +- compat/vcbuild/vcpkg_copy_dlls.bat | 36 ++++ compat/vcbuild/vcpkg_install.bat | 81 +++++++ config.mak.uname | 27 ++- 9 files changed, 375 insertions(+), 184 deletions(-) create mode 100644 compat/vcbuild/vcpkg_copy_dlls.bat create mode 100644 compat/vcbuild/vcpkg_install.bat diff --git a/.gitignore b/.gitignore index ed8d50993b6e01..4e55fc1c375579 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,6 @@ /GIT-BUILD-OPTIONS /GIT-CFLAGS /GIT-LDFLAGS -/GIT-MSVC-GEN /GIT-PREFIX /GIT-PERL-DEFINES /GIT-PYTHON-VARS diff --git a/Makefile b/Makefile index ebd0a75d87b1a5..1ab4b68dce2e0a 100644 --- a/Makefile +++ b/Makefile @@ -2520,8 +2520,6 @@ install: all $(INSTALL) -m 644 $(SCRIPT_LIB) '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' $(INSTALL) $(install_bindir_programs) '$(DESTDIR_SQ)$(bindir_SQ)' ifdef MSVC - $(INSTALL) compat/vcbuild/GEN.DEPS/bin/*.dll '$(DESTDIR_SQ)$(bindir_SQ)' - $(INSTALL) compat/vcbuild/GEN.DEPS/bin/*.pdb '$(DESTDIR_SQ)$(bindir_SQ)' # We DO NOT install the individual foo.o.pdb files because they # have already been rolled up into the exe's pdb file. # We DO NOT have pdb files for the builtin commands (like git-status.exe) @@ -2711,9 +2709,15 @@ endif ifdef MSVC $(RM) $(patsubst %.o,%.o.pdb,$(OBJECTS)) $(RM) $(patsubst %.exe,%.pdb,$(OTHER_PROGRAMS)) + $(RM) $(patsubst %.exe,%.iobj,$(OTHER_PROGRAMS)) + $(RM) $(patsubst %.exe,%.ipdb,$(OTHER_PROGRAMS)) $(RM) $(patsubst %.exe,%.pdb,$(PROGRAMS)) + $(RM) $(patsubst %.exe,%.iobj,$(PROGRAMS)) + $(RM) $(patsubst %.exe,%.ipdb,$(PROGRAMS)) $(RM) $(patsubst %.exe,%.pdb,$(TEST_PROGRAMS)) - $(RM) GIT-MSVC-GEN + $(RM) $(patsubst %.exe,%.iobj,$(TEST_PROGRAMS)) + $(RM) $(patsubst %.exe,%.ipdb,$(TEST_PROGRAMS)) + $(RM) compat/vcbuild/MSVC-DEFS-GEN endif .PHONY: all install profile-clean clean strip diff --git a/compat/vcbuild/.gitignore b/compat/vcbuild/.gitignore index e14691508ec6b5..8f8b794ef39f75 100644 --- a/compat/vcbuild/.gitignore +++ b/compat/vcbuild/.gitignore @@ -1 +1,3 @@ -GEN.* +/vcpkg/ +/MSVC-DEFS-GEN +/VCPKG-DEFS diff --git a/compat/vcbuild/README b/compat/vcbuild/README index df8a6574c9ac24..b986460bb27206 100644 --- a/compat/vcbuild/README +++ b/compat/vcbuild/README @@ -1,3 +1,54 @@ +The Steps to Build Git with VS2015 or VS2017 from the command line. + +1. Install the "vcpkg" open source package manager and build essential + third-party libaries. The steps for this have been captured in a + set of convenience scripts. These can be run from a stock Command + Prompt or from an SDK bash window: + + $ cd + $ ./compat/vcbuild/vcpkg_install.bat + + The vcpkg tools and all of the third-party sources will be installed + in this folder: + /compat/vcbuild/vcpkg/ + + A file will be created with a set of Makefile macros pointing to a + unified "include", "lib", and "bin" directory (release and debug) for + all of the required packages. This file will be included by the main + Makefile: + /compat/vcbuild/MSVC-DEFS-GEN + +2. OPTIONALLY copy the third-party *.dll and *.pdb files into the repo + root to make it easier to run and debug git.exe without having to + manipulate your PATH. This is especially true for debug sessions in + Visual Studio. + + Use ONE of the following forms which should match how you want to + compile git.exe. + + $ ./compat/vcbuild/vcpkg_copy_packages.bat debug + $ ./compat/vcbuild/vcpkg_copy_packages.bat release + +3. Build git using MSVC from an SDK bash window using one of the + following commands: + + $ make MSVC=1 + $ make MSVC=1 DEBUG=1 + +================================================================ + +Alternatively, run `make MSVC=1 vcxproj` and then load the generated +git.sln in Visual Studio. The initial build will install the vcpkg +system and build the dependencies automatically. This will take a while. + +Note that this will automatically add and commit the generated +.sln and .vcxproj files to the repo. You may want to drop this +commit before submitting a Pull Request.... + +Or maybe we should put the .sln/.vcxproj files in the .gitignores +and not do this. I'm not sure. + +================================================================ The Steps of Build Git with VS2008 1. You need the build environment, which contains the Git dependencies diff --git a/compat/vcbuild/find_vs_env.bat b/compat/vcbuild/find_vs_env.bat index 7a736eb3715088..4223ad962cb76c 100644 --- a/compat/vcbuild/find_vs_env.bat +++ b/compat/vcbuild/find_vs_env.bat @@ -1,167 +1,167 @@ -@ECHO OFF -REM ================================================================ -REM You can use either GCC (the default) or MSVC to build git -REM using the GIT-SDK command line tools. -REM $ make -REM $ make MSVC=1 -REM -REM GIT-SDK BASH windows inherit environment variables with all of -REM the bin/lib/include paths for GCC. It DOES NOT inherit values -REM for the corresponding MSVC tools. -REM -REM During normal (non-git) Windows development, you launch one -REM of the provided "developer command prompts" to set environment -REM variables for the MSVC tools. -REM -REM Therefore, to allow MSVC command line builds of git from BASH -REM and MAKE, we must blend these two different worlds. This script -REM attempts to do that. -REM ================================================================ -REM This BAT file starts in a plain (non-developer) command prompt, -REM searches for the "best" commmand prompt setup script, installs -REM it into the current CMD process, and exports the various MSVC -REM environment variables for use by MAKE. -REM -REM The output of this script should be written to a make "include -REM file" and referenced by the top-level Makefile. -REM -REM See "config.mak.uname" (look for GIT-MSVC-GEN). -REM ================================================================ -REM The provided command prompts are custom to each VS release and -REM filled with lots of internal knowledge (such as Registry settings); -REM even their names vary by release, so it is not appropriate for us -REM to look inside them. Rather, just run them in a subordinate -REM process and extract the settings we need. -REM ================================================================ -REM -REM Current (VS2017 and beyond) -REM ------------------- -REM Visual Studio 2017 introduced a new installation layout and -REM support for side-by-side installation of multiple versions of -REM VS2017. Furthermore, these can all coexist with installations -REM of previous versions of VS (which have a completely different -REM layout on disk). -REM -REM VS2017 Update 2 introduced a "vswhere.exe" command: -REM https://github.com/Microsoft/vswhere -REM https://blogs.msdn.microsoft.com/heaths/2017/02/25/vswhere-available/ -REM https://blogs.msdn.microsoft.com/vcblog/2017/03/06/finding-the-visual-c-compiler-tools-in-visual-studio-2017/ -REM -REM VS2015 -REM ------ -REM Visual Studio 2015 uses the traditional VcVarsAll. -REM -REM Earlier Versions -REM ---------------- -REM TODO -REM -REM ================================================================ -REM Note: Throughout this script we use "dir && " rather -REM than "if exist " because of script problems with pathnames -REM containing spaces. -REM ================================================================ - -REM Sanitize PATH to prevent git-sdk paths from confusing "wmic.exe" -REM (called internally in some of the system BAT files). -SET PATH=%SystemRoot%\system32;%SystemRoot%;%SystemRoot%\System32\Wbem; - -REM ================================================================ - -:current - SET vs_where=C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe - dir "%vs_where%" >nul 2>nul && GOTO have_vs_where - GOTO not_2017 - -:have_vs_where - REM Try to use VsWhere to get the location of VsDevCmd. - - REM Keep VsDevCmd from cd'ing away. - SET VSCMD_START_DIR=. - - REM Get the root of the VS product installation. - FOR /F "usebackq tokens=*" %%i IN (`"%vs_where%" -latest -requires Microsoft.VisualStudio.Workload.NativeDesktop -property installationPath`) DO @SET vs_ip=%%i - - SET vs_devcmd=%vs_ip%\Common7\Tools\VsDevCmd.bat - dir "%vs_devcmd%" >nul 2>nul && GOTO have_vs_devcmd - GOTO not_2017 - -:have_vs_devcmd - REM Use VsDevCmd to setup the environment of this process. - REM Setup CL for building 64-bit apps using 64-bit tools. - @call "%vs_devcmd%" -no_logo -arch=x64 -host_arch=x64 - - SET tgt=%VSCMD_ARG_TGT_ARCH% - - SET mn=%VCToolsInstallDir% - SET msvc_includes=-I"%mn%INCLUDE" - SET msvc_libs=-L"%mn%lib\%tgt%" - SET msvc_bin_dir=%mn%bin\Host%VSCMD_ARG_HOST_ARCH%\%tgt% - - SET sdk_dir=%WindowsSdkDir% - SET sdk_ver=%WindowsSDKVersion% - SET si=%sdk_dir%Include\%sdk_ver% - SET sdk_includes=-I"%si%ucrt" -I"%si%um" -I"%si%shared" - SET sl=%sdk_dir%lib\%sdk_ver% - SET sdk_libs=-L"%sl%ucrt\%tgt%" -L"%sl%um\%tgt%" - - SET vs_ver=%VisualStudioVersion% - - GOTO print_vars - -REM ================================================================ - -:not_2017 - REM See if VS2015 is installed. - - SET vs_2015_bat=C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat - dir "%vs_2015_bat%" >nul 2>nul && GOTO have_vs_2015 - GOTO not_2015 - -:have_vs_2015 - REM Use VcVarsAll like the "x64 Native" command prompt. - REM Setup CL for building 64-bit apps using 64-bit tools. - @call "%vs_2015_bat%" amd64 - - REM Note that in VS2015 they use "x64" in some contexts and "amd64" in others. - SET mn=C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\ - SET msvc_includes=-I"%mn%INCLUDE" - SET msvc_libs=-L"%mn%lib\amd64" - SET msvc_bin_dir=%mn%bin\amd64 - - SET sdk_dir=%WindowsSdkDir% - SET sdk_ver=%WindowsSDKVersion% - SET si=%sdk_dir%Include\%sdk_ver% - SET sdk_includes=-I"%si%ucrt" -I"%si%um" -I"%si%shared" -I"%si%winrt" - SET sl=%sdk_dir%lib\%sdk_ver% - SET sdk_libs=-L"%sl%ucrt\x64" -L"%sl%um\x64" - - SET vs_ver=%VisualStudioVersion% - - GOTO print_vars - -REM ================================================================ - -:not_2015 - REM TODO.... - echo TODO support older versions of VS. >&2 - EXIT /B 1 - -REM ================================================================ - -:print_vars - REM Dump the essential vars to stdout to allow the main - REM Makefile to include it. See config.mak.uname. - REM Include DOS-style and BASH-style path for bin dir. - - echo msvc_bin_dir=%msvc_bin_dir% - echo msvc_bin_dir_msys=%msvc_bin_dir:C:=/C% - - echo msvc_includes=%msvc_includes% - echo msvc_libs=%msvc_libs% - - echo sdk_includes=%sdk_includes% - echo sdk_libs=%sdk_libs% - - echo vs_ver=%vs_ver% - - EXIT /B 0 +@ECHO OFF +REM ================================================================ +REM You can use either GCC (the default) or MSVC to build git +REM using the GIT-SDK command line tools. +REM $ make +REM $ make MSVC=1 +REM +REM GIT-SDK BASH windows inherit environment variables with all of +REM the bin/lib/include paths for GCC. It DOES NOT inherit values +REM for the corresponding MSVC tools. +REM +REM During normal (non-git) Windows development, you launch one +REM of the provided "developer command prompts" to set environment +REM variables for the MSVC tools. +REM +REM Therefore, to allow MSVC command line builds of git from BASH +REM and MAKE, we must blend these two different worlds. This script +REM attempts to do that. +REM ================================================================ +REM This BAT file starts in a plain (non-developer) command prompt, +REM searches for the "best" commmand prompt setup script, installs +REM it into the current CMD process, and exports the various MSVC +REM environment variables for use by MAKE. +REM +REM The output of this script should be written to a make "include +REM file" and referenced by the top-level Makefile. +REM +REM See "config.mak.uname" (look for compat/vcbuild/MSVC-DEFS-GEN). +REM ================================================================ +REM The provided command prompts are custom to each VS release and +REM filled with lots of internal knowledge (such as Registry settings); +REM even their names vary by release, so it is not appropriate for us +REM to look inside them. Rather, just run them in a subordinate +REM process and extract the settings we need. +REM ================================================================ +REM +REM Current (VS2017 and beyond) +REM ------------------- +REM Visual Studio 2017 introduced a new installation layout and +REM support for side-by-side installation of multiple versions of +REM VS2017. Furthermore, these can all coexist with installations +REM of previous versions of VS (which have a completely different +REM layout on disk). +REM +REM VS2017 Update 2 introduced a "vswhere.exe" command: +REM https://github.com/Microsoft/vswhere +REM https://blogs.msdn.microsoft.com/heaths/2017/02/25/vswhere-available/ +REM https://blogs.msdn.microsoft.com/vcblog/2017/03/06/finding-the-visual-c-compiler-tools-in-visual-studio-2017/ +REM +REM VS2015 +REM ------ +REM Visual Studio 2015 uses the traditional VcVarsAll. +REM +REM Earlier Versions +REM ---------------- +REM TODO +REM +REM ================================================================ +REM Note: Throughout this script we use "dir && " rather +REM than "if exist " because of script problems with pathnames +REM containing spaces. +REM ================================================================ + +REM Sanitize PATH to prevent git-sdk paths from confusing "wmic.exe" +REM (called internally in some of the system BAT files). +SET PATH=%SystemRoot%\system32;%SystemRoot%;%SystemRoot%\System32\Wbem; + +REM ================================================================ + +:current + SET vs_where=C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe + dir "%vs_where%" >nul 2>nul && GOTO have_vs_where + GOTO not_2017 + +:have_vs_where + REM Try to use VsWhere to get the location of VsDevCmd. + + REM Keep VsDevCmd from cd'ing away. + SET VSCMD_START_DIR=. + + REM Get the root of the VS product installation. + FOR /F "usebackq tokens=*" %%i IN (`"%vs_where%" -latest -requires Microsoft.VisualStudio.Workload.NativeDesktop -property installationPath`) DO @SET vs_ip=%%i + + SET vs_devcmd=%vs_ip%\Common7\Tools\VsDevCmd.bat + dir "%vs_devcmd%" >nul 2>nul && GOTO have_vs_devcmd + GOTO not_2017 + +:have_vs_devcmd + REM Use VsDevCmd to setup the environment of this process. + REM Setup CL for building 64-bit apps using 64-bit tools. + @call "%vs_devcmd%" -no_logo -arch=x64 -host_arch=x64 + + SET tgt=%VSCMD_ARG_TGT_ARCH% + + SET mn=%VCToolsInstallDir% + SET msvc_includes=-I"%mn%INCLUDE" + SET msvc_libs=-L"%mn%lib\%tgt%" + SET msvc_bin_dir=%mn%bin\Host%VSCMD_ARG_HOST_ARCH%\%tgt% + + SET sdk_dir=%WindowsSdkDir% + SET sdk_ver=%WindowsSDKVersion% + SET si=%sdk_dir%Include\%sdk_ver% + SET sdk_includes=-I"%si%ucrt" -I"%si%um" -I"%si%shared" + SET sl=%sdk_dir%lib\%sdk_ver% + SET sdk_libs=-L"%sl%ucrt\%tgt%" -L"%sl%um\%tgt%" + + SET vs_ver=%VisualStudioVersion% + + GOTO print_vars + +REM ================================================================ + +:not_2017 + REM See if VS2015 is installed. + + SET vs_2015_bat=C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat + dir "%vs_2015_bat%" >nul 2>nul && GOTO have_vs_2015 + GOTO not_2015 + +:have_vs_2015 + REM Use VcVarsAll like the "x64 Native" command prompt. + REM Setup CL for building 64-bit apps using 64-bit tools. + @call "%vs_2015_bat%" amd64 + + REM Note that in VS2015 they use "x64" in some contexts and "amd64" in others. + SET mn=C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\ + SET msvc_includes=-I"%mn%INCLUDE" + SET msvc_libs=-L"%mn%lib\amd64" + SET msvc_bin_dir=%mn%bin\amd64 + + SET sdk_dir=%WindowsSdkDir% + SET sdk_ver=%WindowsSDKVersion% + SET si=%sdk_dir%Include\%sdk_ver% + SET sdk_includes=-I"%si%ucrt" -I"%si%um" -I"%si%shared" -I"%si%winrt" + SET sl=%sdk_dir%lib\%sdk_ver% + SET sdk_libs=-L"%sl%ucrt\x64" -L"%sl%um\x64" + + SET vs_ver=%VisualStudioVersion% + + GOTO print_vars + +REM ================================================================ + +:not_2015 + REM TODO.... + echo TODO support older versions of VS. >&2 + EXIT /B 1 + +REM ================================================================ + +:print_vars + REM Dump the essential vars to stdout to allow the main + REM Makefile to include it. See config.mak.uname. + REM Include DOS-style and BASH-style path for bin dir. + + echo msvc_bin_dir=%msvc_bin_dir% + echo msvc_bin_dir_msys=%msvc_bin_dir:C:=/C% + + echo msvc_includes=%msvc_includes% + echo msvc_libs=%msvc_libs% + + echo sdk_includes=%sdk_includes% + echo sdk_libs=%sdk_libs% + + echo vs_ver=%vs_ver% + + EXIT /B 0 diff --git a/compat/vcbuild/scripts/clink.pl b/compat/vcbuild/scripts/clink.pl index b67b8313043352..3d6fa21c1e5690 100755 --- a/compat/vcbuild/scripts/clink.pl +++ b/compat/vcbuild/scripts/clink.pl @@ -14,8 +14,15 @@ my @cflags = (); my @lflags = (); my $is_linking = 0; +my $is_debug = 0; while (@ARGV) { my $arg = shift @ARGV; + if ("$arg" eq "-DDEBUG") { + # Some vcpkg-based libraries have different names for release + # and debug versions. This hack assumes that -DDEBUG comes + # before any "-l*" flags. + $is_debug = 1; + } if ("$arg" =~ /^-[DIMGOZ]/) { push(@cflags, $arg); } elsif ("$arg" eq "-o") { @@ -30,9 +37,13 @@ push(@args, "-Fd$file_out.pdb"); } } elsif ("$arg" eq "-lz") { + if ($is_debug) { + push(@args, "zlibd.lib"); + } else{ push(@args, "zlib.lib"); + } } elsif ("$arg" eq "-liconv") { - push(@args, "iconv.lib"); + push(@args, "libiconv.lib"); } elsif ("$arg" eq "-lcrypto") { push(@args, "libeay32.lib"); } elsif ("$arg" eq "-lssl") { @@ -40,7 +51,7 @@ } elsif ("$arg" eq "-lcurl") { push(@args, "libcurl.lib"); } elsif ("$arg" eq "-lexpat") { - push(@args, "libexpat.lib"); + push(@args, "expat.lib"); } elsif ("$arg" =~ /^-L/ && "$arg" ne "-LTCG") { $arg =~ s/^-L/-LIBPATH:/; push(@lflags, $arg); diff --git a/compat/vcbuild/vcpkg_copy_dlls.bat b/compat/vcbuild/vcpkg_copy_dlls.bat new file mode 100644 index 00000000000000..84409b79452e39 --- /dev/null +++ b/compat/vcbuild/vcpkg_copy_dlls.bat @@ -0,0 +1,36 @@ +@ECHO OFF +REM ================================================================ +REM This script is an optional step. It copies the *.dll and *.pdb +REM files (created by vcpkg_install.bat) into the top-level directory +REM of the repo so that you can type "./git.exe" and find them without +REM having to fixup your PATH. +REM +REM NOTE: Because the names of some DLL files change between DEBUG and +REM NOTE: RELEASE builds when built using "vcpkg.exe", you will need +REM NOTE: to copy up the corresponding version. +REM ================================================================ + + SETLOCAL EnableDelayedExpansion + + @FOR /F "delims=" %%D IN ("%~dp0") DO @SET cwd=%%~fD + cd %cwd% + + SET arch=x64-windows + SET inst=%cwd%vcpkg\installed\%arch% + + IF [%1]==[release] ( + echo Copying RELEASE mode DLLs to repo root... + ) ELSE IF [%1]==[debug] ( + SET inst=%inst%\debug + echo Copying DEBUG mode DLLs to repo root... + ) ELSE ( + echo ERROR: Invalid argument. + echo Usage: %~0 release + echo Usage: %~0 debug + EXIT /B 1 + ) + + xcopy /e/s/v/y %inst%\bin\*.dll ..\..\ + xcopy /e/s/v/y %inst%\bin\*.pdb ..\..\ + + EXIT /B 0 diff --git a/compat/vcbuild/vcpkg_install.bat b/compat/vcbuild/vcpkg_install.bat new file mode 100644 index 00000000000000..3d086c39c31a53 --- /dev/null +++ b/compat/vcbuild/vcpkg_install.bat @@ -0,0 +1,81 @@ +@ECHO OFF +REM ================================================================ +REM This script installs the "vcpkg" source package manager and uses +REM it to build the third-party libraries that git requires when it +REM is built using MSVC. +REM +REM [1] Install VCPKG. +REM [a] Create /compat/vcbuild/vcpkg/ +REM [b] Download "vcpkg". +REM [c] Compile using the currently installed version of VS. +REM [d] Create /compat/vcbuild/vcpkg/vcpkg.exe +REM +REM [2] Install third-party libraries. +REM [a] Download each (which may also install CMAKE). +REM [b] Compile in RELEASE mode and install in: +REM vcpkg/installed//{bin,lib} +REM [c] Compile in DEBUG mode and install in: +REM vcpkg/installed//debug/{bin,lib} +REM [d] Install headers in: +REM vcpkg/installed//include +REM +REM [3] Create a set of MAKE definitions for the top-level +REM Makefile to allow "make MSVC=1" to find the above +REM third-party libraries. +REM [a] Write vcpkg/VCPGK-DEFS +REM +REM https://blogs.msdn.microsoft.com/vcblog/2016/09/19/vcpkg-a-tool-to-acquire-and-build-c-open-source-libraries-on-windows/ +REM https://github.com/Microsoft/vcpkg +REM https://vcpkg.readthedocs.io/en/latest/ +REM ================================================================ + + SETLOCAL EnableDelayedExpansion + + @FOR /F "delims=" %%D IN ("%~dp0") DO @SET cwd=%%~fD + cd %cwd% + + dir vcpkg\vcpkg.exe >nul 2>nul && GOTO :install_libraries + + echo Fetching vcpkg in %cwd%vcpkg + git.exe clone https://github.com/Microsoft/vcpkg vcpkg + IF ERRORLEVEL 1 ( EXIT /B 1 ) + + cd vcpkg + echo Building vcpkg + powershell -exec bypass scripts\bootstrap.ps1 + IF ERRORLEVEL 1 ( EXIT /B 1 ) + + echo Successfully installed %cwd%vcpkg\vcpkg.exe + +:install_libraries + SET arch=x64-windows + + echo Installing third-party libraries... + FOR %%i IN (zlib expat libiconv openssl libssh2 curl) DO ( + cd %cwd%vcpkg + SET p="packages\%%i_%arch%" + IF NOT EXIST "%p%" CALL :sub__install_one %%i + IF ERRORLEVEL 1 ( EXIT /B 1 ) + ) + +:install_defines + cd %cwd% + SET inst=%cwd%vcpkg\installed\%arch% + + echo vcpkg_inc=-I"%inst%\include">VCPKG-DEFS + echo vcpkg_rel_lib=-L"%inst%\lib">>VCPKG-DEFS + echo vcpkg_rel_bin="%inst%\bin">>VCPKG-DEFS + echo vcpkg_dbg_lib=-L"%inst%\debug\lib">>VCPKG-DEFS + echo vcpkg_dbg_bin="%inst%\debug\bin">>VCPKG-DEFS + + EXIT /B 0 + + +:sub__install_one + echo Installing package %1... + + .\vcpkg.exe install %1:%arch% + IF ERRORLEVEL 1 ( EXIT /B 1 ) + + echo Finished %1 + goto :EOF diff --git a/config.mak.uname b/config.mak.uname index 80b248946115e0..3658663a463414 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -18,9 +18,16 @@ ifdef MSVC # Generate and include makefile variables that point to the # currently installed set of MSVC command line tools. -GIT-MSVC-GEN: ./compat/vcbuild/find_vs_env.bat - @./compat/vcbuild/find_vs_env.bat | sed 's|\\|/|g' >GIT-MSVC-GEN --include GIT-MSVC-GEN +compat/vcbuild/MSVC-DEFS-GEN: compat/vcbuild/find_vs_env.bat + @"$<" | tr '\\' / >"$@" +include compat/vcbuild/MSVC-DEFS-GEN + + # See if vcpkg and the vcpkg-build versions of the third-party + # libraries that we use are installed. We include the result + # to get $(vcpkg_*) variables defined for the Makefile. +compat/vcbuild/VCPKG-DEFS: compat/vcbuild/vcpkg_install.bat + @"$<" +include compat/vcbuild/VCPKG-DEFS endif # We choose to avoid "if .. else if .. else .. endif endif" @@ -403,13 +410,13 @@ ifeq ($(uname_S),Windows) EXTLIBS = user32.lib advapi32.lib shell32.lib wininet.lib ws2_32.lib invalidcontinue.obj kernel32.lib ntdll.lib PTHREAD_LIBS = lib = - # Path to the unpacked third-party libraries - MSVC_DEPS = compat/vcbuild/GEN.DEPS - BASIC_CFLAGS += \ - -I$(MSVC_DEPS)/include -I$(MSVC_DEPS)/include/expat -I$(MSVC_DEPS)/include/zlib \ - -L$(MSVC_DEPS)/lib \ - $(sdk_includes) $(sdk_libs) \ - $(msvc_includes) $(msvc_libs) + BASIC_CFLAGS += $(vcpkg_inc) $(sdk_includes) $(msvc_includes) +ifndef DEBUG + BASIC_CFLAGS += $(vcpkg_rel_lib) +else + BASIC_CFLAGS += $(vcpkg_dbg_lib) +endif + BASIC_CFLAGS += $(sdk_libs) $(msvc_libs) # Optionally enable memory leak reporting. # BASIC_CLFAGS += -DUSE_MSVC_CRTDBG From 65ca45152c0acf55a84710cc83c962ed78cb8721 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 19 Sep 2017 12:56:21 -0400 Subject: [PATCH 05/16] msvc: cleanup obsolete nuget files We no longer use NuGet packages... Signed-off-by: Jeff Hostetler Signed-off-by: Johannes Schindelin --- compat/vcbuild/Makefile | 147 ------------------------------- compat/vcbuild/README_VS2015.txt | 54 ------------ compat/vcbuild/nuget.config | 27 ------ compat/vcbuild/packages.config | 21 ----- 4 files changed, 249 deletions(-) delete mode 100644 compat/vcbuild/Makefile delete mode 100644 compat/vcbuild/README_VS2015.txt delete mode 100644 compat/vcbuild/nuget.config delete mode 100644 compat/vcbuild/packages.config diff --git a/compat/vcbuild/Makefile b/compat/vcbuild/Makefile deleted file mode 100644 index 255dd4308435af..00000000000000 --- a/compat/vcbuild/Makefile +++ /dev/null @@ -1,147 +0,0 @@ -## Makefile to install nuget package dependencies. -################################################################## - -INST=GEN.DEPS -INST_INC=$(INST)/include -INST_LIB=$(INST)/lib -INST_BIN=$(INST)/bin - -PKGDIR=GEN.PKGS -NUGET ?= nuget.exe -ifneq ($(shell $(NUGET) help 2>/dev/null; echo $$?),0) - NUGET := /usr/src/build-extra/nuget/nuget.exe -endif - -################################################################## -all: unpack expat libssh libssh_redist curl curl_redist openssl zlib \ - libiconv libiconv_redist - -unpack: - [ -d $(PKGDIR) ] || mkdir $(PKGDIR) - "$(NUGET)" restore packages.config -ConfigFile nuget.config \ - -NonInteractive -OutputDirectory $(PKGDIR) - -insdir: - [ -d $(INST) ] || mkdir $(INST) - [ -d $(INST_INC) ] || mkdir $(INST_INC) - [ -d $(INST_BIN) ] || mkdir $(INST_BIN) - [ -d $(INST_LIB) ] || mkdir $(INST_LIB) - -################################################################## -## We place the expat headers in their own subdirectory. -## The custom is to reference , so compile with: -## -I$(INST_INC)/expat - -EXPAT_VER=2.1.0.11 -EXPAT_ROOT=$(PKGDIR)/expat.$(EXPAT_VER)/build/native -EXPAT_INC=$(EXPAT_ROOT)/include -EXPAT_LIB=$(EXPAT_ROOT)/lib/v110/x64/Release/dynamic/utf8 - -expat: insdir - [ -d $(INST_INC)/expat ] || mkdir $(INST_INC)/expat - cp -r $(EXPAT_INC)/* $(INST_INC)/expat/ - cp -r $(EXPAT_LIB)/* $(INST_LIB)/ - - -################################################################## - -LIBICONV_VER=1.14.0.11 -LIBICONV_ROOT=$(PKGDIR)/libiconv.$(LIBICONV_VER)/build/native -LIBICONV_INC=$(LIBICONV_ROOT)/include -LIBICONV_LIB=$(LIBICONV_ROOT)/lib/v110/x64/Release/dynamic/cdecl - -libiconv: insdir - cp -r $(LIBICONV_INC)/* $(INST_INC)/ - cp -r $(LIBICONV_LIB)/libiconv.lib $(INST_LIB)/iconv.lib - - -LIBICONV_REDIST_ROOT=$(PKGDIR)/libiconv.redist.$(LIBICONV_VER)/build/native -LIBICONV_REDIST_BIN=$(LIBICONV_REDIST_ROOT)/bin/v110/x64/Release/dynamic/cdecl - -libiconv_redist: insdir - cp -r $(LIBICONV_REDIST_BIN)/* $(INST_BIN)/ - - -################################################################## - -LIBSSH_VER=1.4.3.3 -LIBSSH_ROOT=$(PKGDIR)/libssh2.$(LIBSSH_VER)/build/native -LIBSSH_INC=$(LIBSSH_ROOT)/include -LIBSSH_LIB=$(LIBSSH_ROOT)/lib/v110/x64/Release/dynamic/cdecl - -libssh: insdir - [ -d $(INST_INC)/libssh2 ] || mkdir $(INST_INC)/libssh2 - cp -r $(LIBSSH_INC)/* $(INST_INC)/libssh2 - cp -r $(LIBSSH_LIB)/* $(INST_LIB)/ - - -LIBSSH_REDIST_ROOT=$(PKGDIR)/libssh2.redist.$(LIBSSH_VER)/build/native -LIBSSH_REDIST_BIN=$(LIBSSH_REDIST_ROOT)/bin/v110/x64/Release/dynamic/cdecl - -libssh_redist: insdir - cp -r $(LIBSSH_REDIST_BIN)/* $(INST_BIN)/ - - -################################################################## -## We place the curl headers in their own subdirectory. -## The custom is to reference , so compile with: -## -I$(INST_INC) - -CURL_VER=7.30.0.2 -CURL_ROOT=$(PKGDIR)/curl.$(CURL_VER)/build/native -CURL_INC=$(CURL_ROOT)/include/curl -CURL_LIB=$(CURL_ROOT)/lib/v110/x64/Release/dynamic - -curl: insdir - [ -d $(INST_INC)/curl ] || mkdir $(INST_INC)/curl - cp -r $(CURL_INC)/* $(INST_INC)/curl - cp -r $(CURL_LIB)/* $(INST_LIB)/ - - -CURL_REDIST_ROOT=$(PKGDIR)/curl.redist.$(CURL_VER)/build/native -CURL_REDIST_BIN=$(CURL_REDIST_ROOT)/bin/v110/x64/Release/dynamic - -curl_redist: insdir - cp -r $(CURL_REDIST_BIN)/* $(INST_BIN)/ - - -################################################################## -## We place the openssl headers in their own subdirectory. -## The custom is to reference , so compile with: -## -I$(INST_INC) - -OPENSSL_VER=1.0.2.1 -OPENSSL_ROOT=$(PKGDIR)/openssl.v140.windesktop.msvcstl.dyn.rt-dyn.x64.$(OPENSSL_VER) -OPENSSL_INC=$(OPENSSL_ROOT)/build/native/include/openssl -OPENSSL_LIB=$(OPENSSL_ROOT)/lib/native/v140/windesktop/msvcstl/dyn/rt-dyn/x64/release - -openssl: insdir - [ -d $(INST_INC)/openssl ] || mkdir $(INST_INC)/openssl - cp -r $(OPENSSL_INC)/* $(INST_INC)/openssl - cp -r $(OPENSSL_LIB)/*.lib $(INST_LIB)/ - cp -r $(OPENSSL_LIB)/*.dll $(INST_BIN)/ - cp -r $(OPENSSL_LIB)/*.pdb $(INST_BIN)/ - -################################################################## -## We place the zlib headers in their own subdirectory. -## The custom is to reference , so compile with: -## -I$(INST_INC)/zlib - -ZLIB_VER=1.2.8.8 -ZLIB_ROOT=$(PKGDIR)/zlib.v140.windesktop.msvcstl.dyn.rt-dyn.$(ZLIB_VER) -ZLIB_INC=$(ZLIB_ROOT)/build/native/include -ZLIB_LIB=$(ZLIB_ROOT)/lib/native/v140/windesktop/msvcstl/dyn/rt-dyn/x64/RelWithDebInfo - -zlib: insdir - [ -d $(INST_INC)/zlib ] || mkdir $(INST_INC)/zlib - cp -r $(ZLIB_INC)/* $(INST_INC)/zlib - cp -r $(ZLIB_LIB)/*.lib $(INST_LIB)/ - cp -r $(ZLIB_LIB)/*.dll $(INST_BIN)/ - cp -r $(ZLIB_LIB)/*.pdb $(INST_BIN)/ - -################################################################## -clean: - rm -rf $(INST) - -clobber: clean - rm -rf $(PKGDIR) diff --git a/compat/vcbuild/README_VS2015.txt b/compat/vcbuild/README_VS2015.txt deleted file mode 100644 index a9c6f59dc6cfd8..00000000000000 --- a/compat/vcbuild/README_VS2015.txt +++ /dev/null @@ -1,54 +0,0 @@ -Instructions for building Git for Windows using VS2015. -================================================================ - -Installing third-party dependencies: -==================================== - -[1] Install nuget.exe somewhere on your system and add it to your PATH. - https://docs.nuget.org/consume/command-line-reference - https://dist.nuget.org/index.html - -[2] Download required nuget packages for third-party libraries. - Using a terminal window, type: - - make -C compat/vcbuild - - This will download the packages, unpack them into GEN.PKGS, - and populate the {include, lib, bin} directories in GEN.DEPS. - - -Building Git for Windows using VS2015: -====================================== - -[3] Build 64-bit version of Git for Windows. - Using a terminal window: - - make MSVC=1 DEBUG=1 - - -[4] Add compat/vcbuild/GEN.DEPS/bin to your PATH. - -[5] You should then be able to run the test suite and any interactive - commands. - -[6] To debug/profile in VS, open the git.exe in VS and run/debug - it. (Be sure to add GEN.DEPS/bin to the PATH in the debug - dialog.) - - -TODO List: -========== - -[A] config.mak.uname currently contains hard-coded paths - to the various MSVC and SDK libraries for the 64-bit - version of the compilers and libaries. - - See: SANE_TOOL_PATH, MSVC_DEPS, MSVC_SDK*, MSVC_VCDIR. - - Long term, we need to figure out how to properly import - values for %VCINSTALLDIR%, %LIB%, %LIBPATH%, and the - other values normally set by "vsvars32.bat" when a - developer command prompt is started. This would also - allow us to switch between 32- and 64-bit tool chains. - -[B] We need to build SLN or VCPROJ files. diff --git a/compat/vcbuild/nuget.config b/compat/vcbuild/nuget.config deleted file mode 100644 index b5e734950478fa..00000000000000 --- a/compat/vcbuild/nuget.config +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - diff --git a/compat/vcbuild/packages.config b/compat/vcbuild/packages.config deleted file mode 100644 index 53b1ede4f5361c..00000000000000 --- a/compat/vcbuild/packages.config +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - From ce94fabb3fd50adda455688e1259269dfff2a6b2 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 19 Sep 2017 12:56:21 -0400 Subject: [PATCH 06/16] msvc: get rid of the MSVC_DEPS variable As we do not consume NuGet packages any longer, there is no sense to try to point PATH to their unpacked .dll files, either. Signed-off-by: Jeff Hostetler Signed-off-by: Johannes Schindelin --- Makefile | 3 --- t/test-lib.sh | 3 --- 2 files changed, 6 deletions(-) diff --git a/Makefile b/Makefile index 1ab4b68dce2e0a..1602458f588325 100644 --- a/Makefile +++ b/Makefile @@ -2382,9 +2382,6 @@ ifdef GIT_INTEROP_MAKE_OPTS endif ifdef TEST_GIT_INDEX_VERSION @echo TEST_GIT_INDEX_VERSION=\''$(subst ','\'',$(subst ','\'',$(TEST_GIT_INDEX_VERSION)))'\' >>$@+ -endif -ifdef MSVC_DEPS - @echo MSVC_DEPS=\''$(subst ','\'',$(subst ','\'',$(MSVC_DEPS)))'\' >>$@+ endif @if cmp $@+ $@ >/dev/null 2>&1; then $(RM) $@+; else mv $@+ $@; fi diff --git a/t/test-lib.sh b/t/test-lib.sh index 9a31e129936414..dd1b4252197153 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -62,9 +62,6 @@ fi . "$GIT_BUILD_DIR"/GIT-BUILD-OPTIONS export PERL_PATH SHELL_PATH -test -z "$MSVC_DEPS" || -PATH="$GIT_BUILD_DIR/$MSVC_DEPS/bin$PATH_SEP$PATH" - ################################################################ # It appears that people try to run tests without building... test -n "$GIT_TEST_INSTALLED" || "$GIT_BUILD_DIR/git$X" >/dev/null || From d9cc4631b65aa997e3fed555e4881cb180ebb0d9 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 29 Nov 2017 21:43:04 +0100 Subject: [PATCH 07/16] vcxproj: let vcs-svn depend on libgit It really does depend on libgit. It does not hurt to let it depend on xdiff, and it makes the code simpler. It is necessary to get this dependency chain right, because we will introduce a change where the vcpkg system is initialized before building libgit. The vcpkg system will then build the dependencies needed by Git (and thereby make the include headers available): As the vcpkg system cannot be run in parallel (it does not lock, wreaking havoc with files being accessed and written at the same time, letting the vcpkg processes stumble over each others' toes. We prevent that by ensuring that only one project is built at first: libgit. And this project's PreBuildEvent will be used to initialize vcpkg and build all dependencies. Subsequently, the other projects can be built in parallel. Signed-off-by: Johannes Schindelin --- contrib/buildsystems/Generators/Vcxproj.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/buildsystems/Generators/Vcxproj.pm b/contrib/buildsystems/Generators/Vcxproj.pm index ae443449aac203..8aebe96c2c1b90 100644 --- a/contrib/buildsystems/Generators/Vcxproj.pm +++ b/contrib/buildsystems/Generators/Vcxproj.pm @@ -296,7 +296,7 @@ EOM print F << "EOM"; EOM - if (!$static_library) { + if (!$static_library || $target eq 'vcs-svn') { my $uuid_libgit = $$build_structure{"LIBS_libgit_GUID"}; my $uuid_xdiff_lib = $$build_structure{"LIBS_xdiff/lib_GUID"}; From 04f2fb66596d1f6d6458f319ee6af574e9f24f75 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 29 Nov 2017 22:06:09 +0100 Subject: [PATCH 08/16] vcxproj: do not use " unnecessarily The .vcxproj's text nodes do not actually need to URL-encode double quotes. So let's not do that. Signed-off-by: Johannes Schindelin --- contrib/buildsystems/Generators/Vcxproj.pm | 2 -- 1 file changed, 2 deletions(-) diff --git a/contrib/buildsystems/Generators/Vcxproj.pm b/contrib/buildsystems/Generators/Vcxproj.pm index 8aebe96c2c1b90..65fe56d2742ad7 100644 --- a/contrib/buildsystems/Generators/Vcxproj.pm +++ b/contrib/buildsystems/Generators/Vcxproj.pm @@ -123,7 +123,6 @@ sub createProject { my $defines = join(";", sort(@{$$build_structure{"$prefix${name}_DEFINES"}})); my $includes= join(";", sort(map { s/^-I//; s/\//\\/g; File::Spec->file_name_is_absolute($_) ? $_ : "$rel_dir\\$_" } @{$$build_structure{"$prefix${name}_INCLUDES"}})); my $cflags = join(" ", sort(map { s/^-[GLMOZ].*//; s/.* .*/"$&"/; $_; } @{$$build_structure{"$prefix${name}_CFLAGS"}})); - $cflags =~ s/\"/"/g; $cflags =~ s//>/g; @@ -133,7 +132,6 @@ sub createProject { } $defines =~ s/-D//g; - $defines =~ s/\"/"/g; $defines =~ s//>/g; $defines =~ s/\'//g; From 77a6527f54942ff444a00bd5be08abbfb23cf57a Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 28 Nov 2017 18:29:18 +0100 Subject: [PATCH 09/16] vcxproj: automatically initialize the vcpkg system We just introduced a way to build Git for Windows with MSVC on the command line using vcpkg-generated, up-to-date dependencies. Let's bring that convenience to the Visual Studio project, too. (The previous method, fetching NuGet packages, is fraught with problems: as C++ libraries have to be built for every architecture and for every toolset, the NuGet packages which we would like to consume fell behind and are not up-to-date with the current versions of the libraries, e.g. cURL and OpenSSL. By using vcpkg we avoid that problem, always building the newest dependency versions.) The trick is to initialize the VCPKG system once, and then build Git's dependencies using it. We do that by attaching a pre-build event to the libgit project (which is now the base project on which all others depend, therefore no other project is built in paralleli, side-stepping issues with vcpkg being unprepared for being run in parallel). Signed-off-by: Johannes Schindelin --- config.mak.uname | 2 +- contrib/buildsystems/Generators/Vcxproj.pm | 89 +++++++++------------- contrib/buildsystems/engine.pl | 2 + 3 files changed, 37 insertions(+), 56 deletions(-) diff --git a/config.mak.uname b/config.mak.uname index 3658663a463414..c7e22726c49134 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -450,7 +450,7 @@ vcxproj: # Make .vcxproj files and add them unset QUIET_GEN QUIET_BUILT_IN; \ perl contrib/buildsystems/generate -g Vcxproj - git add -f git.sln {*,*/lib,t/helper/*}/{packages.config,*.vcxproj} + git add -f git.sln {*,*/lib,t/helper/*}/*.vcxproj # Add common-cmds.h $(MAKE) MSVC=1 prefix=/mingw64 common-cmds.h diff --git a/contrib/buildsystems/Generators/Vcxproj.pm b/contrib/buildsystems/Generators/Vcxproj.pm index 65fe56d2742ad7..53d2f4dce10d39 100644 --- a/contrib/buildsystems/Generators/Vcxproj.pm +++ b/contrib/buildsystems/Generators/Vcxproj.pm @@ -126,9 +126,12 @@ sub createProject { $cflags =~ s//>/g; - my $libs = ''; + my $libs_release = ''; + my $libs_debug = ''; if (!$static_library) { - $libs = join(";", sort(grep /^(?!libgit\.lib|xdiff\/lib\.lib|vcs-svn\/lib\.lib|libcurl\.lib|libeay32\.lib|libiconv\.lib|ssleay32\.lib|zlib\.lib)/, @{$$build_structure{"$prefix${name}_LIBS"}})); + $libs_release = join(";", sort(grep /^(?!libgit\.lib|xdiff\/lib\.lib|vcs-svn\/lib\.lib)/, @{$$build_structure{"$prefix${name}_LIBS"}})); + $libs_debug = $libs_release; + $libs_debug =~ s/zlib\.lib/zlibd\.lib/; } $defines =~ s/-D//g; @@ -138,45 +141,6 @@ sub createProject { die "Could not create the directory $target for $label project!\n" unless (-d "$target" || mkdir "$target"); - use File::Copy; - copy("$git_dir/compat/vcbuild/packages.config", "$target/packages.config"); - - my $needsCurl = grep(/libcurl.lib/, @{$$build_structure{"$prefix${name}_LIBS"}}); - my $targetsImport = ''; - my $targetsErrors = ''; - my $afterTargets = ''; - open F, "<$git_dir/compat/vcbuild/packages.config"; - while () { - if (/"; - } elsif ($needsCurl && $1 eq 'curl') { - # libcurl is only available targeting v100 and v110 - $libs .= ";$rel_dir\\compat\\vcbuild\\GEN.PKGS\\$1.$2\\build\\native\\lib\\v110\\\$(Platform)\\Release\\dynamic\\libcurl.lib"; - $afterTargets .= "\n "; - } elsif ($needsCurl && $1 eq 'expat') { - # libexpat is only available targeting v100 and v110 - $libs .= ";$rel_dir\\compat\\vcbuild\\GEN.PKGS\\$1.$2\\build\\native\\lib\\v110\\\$(Platform)\\Release\\dynamic\\utf8\\libexpat.lib"; - } elsif ($1 eq 'zlib') { - # zlib - $libs .= ";$rel_dir\\compat\\vcbuild\\GEN.PKGS\\$1.v140.windesktop.msvcstl.dyn.rt-dyn.$2\\lib\\native\\v140\\windesktop\\msvcstl\\dyn\\rt-dyn\\x64\\RelWithDebInfo\\zlib.lib"; - } elsif ($1 eq 'openssl') { - # openssl - $libs .= ";$rel_dir\\compat\\vcbuild\\GEN.PKGS\\$1.v140.windesktop.msvcstl.dyn.rt-dyn.x64.$2\\lib\\native\\v140\\windesktop\\msvcstl\\dyn\\rt-dyn\\x64\\release\\libeay32.lib"; - $libs .= ";$rel_dir\\compat\\vcbuild\\GEN.PKGS\\$1.v140.windesktop.msvcstl.dyn.rt-dyn.x64.$2\\lib\\native\\v140\\windesktop\\msvcstl\\dyn\\rt-dyn\\x64\\release\\ssleay32.lib"; - } - next if ($1 =~ /^(zlib$|openssl(?!.*(x64|x86)$))/); - my $targetsFile = "$rel_dir\\compat\\vcbuild\\GEN.PKGS\\$1.$2\\build\\native\\$1.targets"; - $targetsImport .= "\n "; - $targetsErrors .= "\n "; - } - } - close F; - open F, ">$vcxproj" or die "Could not open $vcxproj for writing!\n"; binmode F, ":crlf :utf8"; print F chr(0xFEFF); @@ -204,6 +168,16 @@ sub createProject { $uuid Win32Proj + x86-windows + x64-windows + $cdup\\compat\\vcbuild\\vcpkg\\installed\\\$(VCPKGArch) + \$(VCPKGArchDirectory)\\debug\\bin + \$(VCPKGArchDirectory)\\debug\\lib + \$(VCPKGArchDirectory)\\bin + \$(VCPKGArchDirectory)\\lib + \$(VCPKGArchDirectory)\\include + $libs_debug + $libs_release @@ -237,7 +211,7 @@ sub createProject { $cflags %(AdditionalOptions) - $includes;%(AdditionalIncludeDirectories) + $cdup;$cdup\\compat;$cdup\\compat\\regex;$cdup\\compat\\win32;$cdup\\compat\\poll;$cdup\\compat\\vcbuild\\include;\$(VCPKGIncludeDirectory);%(AdditionalIncludeDirectories) true OnlyExplicitInline @@ -248,11 +222,22 @@ sub createProject { true - $libs;\$(AdditionalDependencies) + \$(VCPKGLibDirectory);%(AdditionalLibraryDirectories) + \$(VCPKGLibs);\$(AdditionalDependencies) invalidcontinue.obj %(AdditionalOptions) $cdup\\compat\\win32\\git.manifest Console +EOM + if ($target eq 'libgit') { + print F << "EOM"; + + Initialize VCPKG + call "$cdup\\compat\\vcbuild\\vcpkg_install.bat" + +EOM + } + print F << "EOM"; @@ -323,21 +308,15 @@ EOM EOM } print F << "EOM"; - - - - $targetsImport - - - - This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - $targetsErrors - EOM - if (!$static_library && $afterTargets ne '') { + if (!$static_library) { print F << "EOM"; - $afterTargets + + + + + EOM } diff --git a/contrib/buildsystems/engine.pl b/contrib/buildsystems/engine.pl index 9f4e7a2ccb9352..8bb07e8e25cc50 100755 --- a/contrib/buildsystems/engine.pl +++ b/contrib/buildsystems/engine.pl @@ -347,6 +347,8 @@ sub handleLinkLine push(@libs, "ssleay32.lib"); } elsif ("$part" eq "-lcurl") { push(@libs, "libcurl.lib"); + } elsif ("$part" eq "-lexpat") { + push(@libs, "expat.lib"); } elsif ("$part" eq "-liconv") { push(@libs, "libiconv.lib"); } elsif ($part =~ /^[-\/]/) { From 01bbbed1f467b68ae1f39188913ab2868bcd685c Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 5 Dec 2017 12:50:57 +0100 Subject: [PATCH 10/16] vcxproj: move `vcxproj` target outside the MSVC block The `vcxproj` target does not, in fact, depend on MSVC being defined, so let's just move it outside of that block. Signed-off-by: Johannes Schindelin --- config.mak.uname | 118 +++++++++++++++++++++++------------------------ 1 file changed, 59 insertions(+), 59 deletions(-) diff --git a/config.mak.uname b/config.mak.uname index c7e22726c49134..ea572bbfd41c94 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -440,65 +440,6 @@ endif X = .exe compat/msvc.o: compat/msvc.c compat/mingw.c GIT-CFLAGS - -vcxproj: - # Require clean work tree - git update-index -q --refresh && \ - git diff-files --quiet && \ - git diff-index --cached --quiet HEAD -- - - # Make .vcxproj files and add them - unset QUIET_GEN QUIET_BUILT_IN; \ - perl contrib/buildsystems/generate -g Vcxproj - git add -f git.sln {*,*/lib,t/helper/*}/*.vcxproj - - # Add common-cmds.h - $(MAKE) MSVC=1 prefix=/mingw64 common-cmds.h - git add -f common-cmds.h - - # Add scripts - rm -f perl/perl.mak - $(MAKE) MSVC=1 prefix=/mingw64 \ - $(SCRIPT_LIB) $(SCRIPT_SH_GEN) $(SCRIPT_PERL_GEN) - # Strip out the sane tool path, needed only for building - sed -i '/^git_broken_path_fix ".*/d' git-sh-setup - git add -f $(SCRIPT_LIB) $(SCRIPT_SH_GEN) $(SCRIPT_PERL_GEN) - - # Add Perl module - $(MAKE) -C perl - git add -f perl/blib/lib - - # Add bin-wrappers, for testing - rm -rf bin-wrappers/ - $(MAKE) MSVC=1 prefix=/mingw64 $(test_bindir_programs) - # Ensure that the GIT_EXEC_PATH is a Unix-y one, and that the absolute - # path of the repository is not hard-coded (GIT_EXEC_PATH will be set - # by test-lib.sh according to the current setup) - sed -i -e 's/^\(GIT_EXEC_PATH\)=.*/test -n "$${\1##*:*}" ||\ - \1="$$(cygpath -u "$$\1")"/' \ - -e "s|'$$(pwd)|\"\$$GIT_EXEC_PATH\"'|g" bin-wrappers/* - # Ensure that test-* helpers find the .dll files copied to top-level - sed -i 's|^PATH=.*|&:"$$GIT_EXEC_PATH"|' bin-wrappers/test-* - # We do not want to force hard-linking builtins - sed -i 's|\(git\)-\([-a-z]*\)\.exe"|\1.exe" \2|g' \ - bin-wrappers/git-{receive-pack,upload-archive} - git add -f $(test_bindir_programs) - # remote-ext is a builtin, but invoked as if it were external - sed 's|receive-pack|remote-ext|g' \ - bin-wrappers/git-remote-ext - git add -f bin-wrappers/git-remote-ext - - # Add templates - $(MAKE) -C templates - git add -f templates/boilerplates.made templates/blt/ - - # Add build options - $(MAKE) MSVC=1 prefix=/mingw64 GIT-BUILD-OPTIONS - git add -f GIT-BUILD-OPTIONS - - # Commit the whole shebang - git commit -m "Generate Visual Studio solution" \ - -m "Auto-generated by \`$(MAKE)$(MAKEFLAGS) $@\`" endif ifeq ($(uname_S),Interix) NO_INITGROUPS = YesPlease @@ -759,3 +700,62 @@ ifeq ($(uname_S),QNX) NO_STRCASESTR = YesPlease NO_STRLCPY = YesPlease endif + +vcxproj: + # Require clean work tree + git update-index -q --refresh && \ + git diff-files --quiet && \ + git diff-index --cached --quiet HEAD -- + + # Make .vcxproj files and add them + unset QUIET_GEN QUIET_BUILT_IN; \ + perl contrib/buildsystems/generate -g Vcxproj + git add -f git.sln {*,*/lib,t/helper/*}/*.vcxproj + + # Add common-cmds.h + $(MAKE) MSVC=1 prefix=/mingw64 common-cmds.h + git add -f common-cmds.h + + # Add scripts + rm -f perl/perl.mak + $(MAKE) MSVC=1 prefix=/mingw64 \ + $(SCRIPT_LIB) $(SCRIPT_SH_GEN) $(SCRIPT_PERL_GEN) + # Strip out the sane tool path, needed only for building + sed -i '/^git_broken_path_fix ".*/d' git-sh-setup + git add -f $(SCRIPT_LIB) $(SCRIPT_SH_GEN) $(SCRIPT_PERL_GEN) + + # Add Perl module + $(MAKE) -C perl + git add -f perl/blib/lib + + # Add bin-wrappers, for testing + rm -rf bin-wrappers/ + $(MAKE) MSVC=1 prefix=/mingw64 $(test_bindir_programs) + # Ensure that the GIT_EXEC_PATH is a Unix-y one, and that the absolute + # path of the repository is not hard-coded (GIT_EXEC_PATH will be set + # by test-lib.sh according to the current setup) + sed -i -e 's/^\(GIT_EXEC_PATH\)=.*/test -n "$${\1##*:*}" ||\ + \1="$$(cygpath -u "$$\1")"/' \ + -e "s|'$$(pwd)|\"\$$GIT_EXEC_PATH\"'|g" bin-wrappers/* + # Ensure that test-* helpers find the .dll files copied to top-level + sed -i 's|^PATH=.*|&:"$$GIT_EXEC_PATH"|' bin-wrappers/test-* + # We do not want to force hard-linking builtins + sed -i 's|\(git\)-\([-a-z]*\)\.exe"|\1.exe" \2|g' \ + bin-wrappers/git-{receive-pack,upload-archive} + git add -f $(test_bindir_programs) + # remote-ext is a builtin, but invoked as if it were external + sed 's|receive-pack|remote-ext|g' \ + bin-wrappers/git-remote-ext + git add -f bin-wrappers/git-remote-ext + + # Add templates + $(MAKE) -C templates + git add -f templates/boilerplates.made templates/blt/ + + # Add build options + $(MAKE) MSVC=1 prefix=/mingw64 GIT-BUILD-OPTIONS + git add -f GIT-BUILD-OPTIONS + + # Commit the whole shebang + git commit -m "Generate Visual Studio solution" \ + -m "Auto-generated by \`$(MAKE)$(MAKEFLAGS) $@\`" From 143ce923e2bc4874da54f59193d1fda9ea59a0a4 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 1 Dec 2017 21:00:00 +0100 Subject: [PATCH 11/16] Allow `make vcxproj` without initializing vcpkg The idea of the `vcxproj` target is to generate .sln/.vcxproj files and then commit them, to be used elsewhere. Typically, this is done in a VSTS job whenever `master` changes. So there is little use in initializing vcpkg and building all the dependencies: they are not necessary here. Signed-off-by: Johannes Schindelin --- config.mak.uname | 10 ++++++---- contrib/buildsystems/engine.pl | 3 ++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/config.mak.uname b/config.mak.uname index ea572bbfd41c94..26f77c36ec175a 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -25,10 +25,12 @@ include compat/vcbuild/MSVC-DEFS-GEN # See if vcpkg and the vcpkg-build versions of the third-party # libraries that we use are installed. We include the result # to get $(vcpkg_*) variables defined for the Makefile. +ifeq (,$(SKIP_VCPKG)) compat/vcbuild/VCPKG-DEFS: compat/vcbuild/vcpkg_install.bat @"$<" include compat/vcbuild/VCPKG-DEFS endif +endif # We choose to avoid "if .. else if .. else .. endif endif" # because maintaining the nesting to match is a pain. If @@ -713,12 +715,12 @@ vcxproj: git add -f git.sln {*,*/lib,t/helper/*}/*.vcxproj # Add common-cmds.h - $(MAKE) MSVC=1 prefix=/mingw64 common-cmds.h + $(MAKE) MSVC=1 SKIP_VCPKG=1 prefix=/mingw64 common-cmds.h git add -f common-cmds.h # Add scripts rm -f perl/perl.mak - $(MAKE) MSVC=1 prefix=/mingw64 \ + $(MAKE) MSVC=1 SKIP_VCPKG=1 prefix=/mingw64 \ $(SCRIPT_LIB) $(SCRIPT_SH_GEN) $(SCRIPT_PERL_GEN) # Strip out the sane tool path, needed only for building sed -i '/^git_broken_path_fix ".*/d' git-sh-setup @@ -730,7 +732,7 @@ vcxproj: # Add bin-wrappers, for testing rm -rf bin-wrappers/ - $(MAKE) MSVC=1 prefix=/mingw64 $(test_bindir_programs) + $(MAKE) MSVC=1 SKIP_VCPKG=1 prefix=/mingw64 $(test_bindir_programs) # Ensure that the GIT_EXEC_PATH is a Unix-y one, and that the absolute # path of the repository is not hard-coded (GIT_EXEC_PATH will be set # by test-lib.sh according to the current setup) @@ -753,7 +755,7 @@ vcxproj: git add -f templates/boilerplates.made templates/blt/ # Add build options - $(MAKE) MSVC=1 prefix=/mingw64 GIT-BUILD-OPTIONS + $(MAKE) MSVC=1 SKIP_VCPKG=1 prefix=/mingw64 GIT-BUILD-OPTIONS git add -f GIT-BUILD-OPTIONS # Commit the whole shebang diff --git a/contrib/buildsystems/engine.pl b/contrib/buildsystems/engine.pl index 8bb07e8e25cc50..fba8a3f056a0e1 100755 --- a/contrib/buildsystems/engine.pl +++ b/contrib/buildsystems/engine.pl @@ -82,7 +82,8 @@ sub showUsage # Capture the make dry stderr to file for review (will be empty for a release build). my $ErrsFile = "msvc-build-makedryerrors.txt"; -@makedry = `make -C $git_dir -n MSVC=1 V=1 2>$ErrsFile` if !@makedry; +@makedry = `make -C $git_dir -n MSVC=1 SKIP_VCPKG=1 V=1 2>$ErrsFile` +if !@makedry; # test for an empty Errors file and remove it unlink $ErrsFile if -f -z $ErrsFile; From 42cbbf76e5f95e87f930900b69a1711fa9d27f4f Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 19 Dec 2017 13:54:14 +0100 Subject: [PATCH 12/16] vcxproj: also link-or-copy builtins The problem with not having, say, git-receive-pack.exe after a full build is that the test suite will then happily use the *installed* git-receive-pack.exe because it finds nothing else. Absolutely not what we want. We want to have confidence that our test covers the MSVC-built Git executables, and not some random stuff. Signed-off-by: Johannes Schindelin --- config.mak.uname | 14 ++++++++++++++ contrib/buildsystems/Generators/Vcxproj.pm | 3 +++ 2 files changed, 17 insertions(+) diff --git a/config.mak.uname b/config.mak.uname index 26f77c36ec175a..6deca782f8abdb 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -709,6 +709,20 @@ vcxproj: git diff-files --quiet && \ git diff-index --cached --quiet HEAD -- + # Generate the LinkOrCopyBuiltins.targets file + (echo '' && \ + echo ' ' && \ + for name in $(BUILT_INS);\ + do \ + echo ' '; \ + done && \ + for name in $(REMOTE_CURL_ALIASES); \ + do \ + echo ' '; \ + done && \ + echo ' ' && \ + echo '') >git/LinkOrCopyBuiltins.targets + # Make .vcxproj files and add them unset QUIET_GEN QUIET_BUILT_IN; \ perl contrib/buildsystems/generate -g Vcxproj diff --git a/contrib/buildsystems/Generators/Vcxproj.pm b/contrib/buildsystems/Generators/Vcxproj.pm index 53d2f4dce10d39..acd60acb22d58d 100644 --- a/contrib/buildsystems/Generators/Vcxproj.pm +++ b/contrib/buildsystems/Generators/Vcxproj.pm @@ -320,6 +320,9 @@ EOM EOM } + if ($target eq 'git') { + print F " \n"; + } print F << "EOM"; EOM From e696826e2754c7f08f5f268519c44cd80979f012 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 8 Dec 2017 12:10:58 -0500 Subject: [PATCH 13/16] Makefile: add third-party DLLs to install target Signed-off-by: Jeff Hostetler --- Makefile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Makefile b/Makefile index 1602458f588325..f5f16dfc78cbef 100644 --- a/Makefile +++ b/Makefile @@ -2535,6 +2535,13 @@ ifdef MSVC $(INSTALL) git-remote-testsvn.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' $(INSTALL) git-sh-i18n--envsubst.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' $(INSTALL) git-show-index.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' +ifndef DEBUG + $(INSTALL) $(vcpkg_rel_bin)/*.dll '$(DESTDIR_SQ)$(bindir_SQ)' + $(INSTALL) $(vcpkg_rel_bin)/*.pdb '$(DESTDIR_SQ)$(bindir_SQ)' +else + $(INSTALL) $(vcpkg_dbg_bin)/*.dll '$(DESTDIR_SQ)$(bindir_SQ)' + $(INSTALL) $(vcpkg_dbg_bin)/*.pdb '$(DESTDIR_SQ)$(bindir_SQ)' +endif endif $(MAKE) -C templates DESTDIR='$(DESTDIR_SQ)' install $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(mergetools_instdir_SQ)' From cd8ea9aa7a078a5d19906bf5088a4fb50cea60ef Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Sun, 17 Sep 2017 12:16:55 +0900 Subject: [PATCH 14/16] gc: call fscanf() with %s, not %c, when reading hostname Earlier in this codepath, we (ab)used "%c" to read the hostname recorded in the lockfile into locking_host[HOST_NAME_MAX + 1] while substituting with the actual value of HOST_NAME_MAX. This turns out to be incorrect, as it is an instruction to read exactly the specified number of bytes. Because we are trying to read at most that many bytes, we should be using "%s" instead. [jes: this is a backport of afe2fab72c3 (gc: call fscanf() with %s, not %c, when reading hostname, 2017-09-17) to the vs2017_vcpkg branch, as it turns out that MSVC's fscanf() is more correct than GNU libc's, and returns only 1 correctly scanned field instead of the 2 expected by Git, making t6500-gc.sh fail.] Helped-by: A. Wilcox Signed-off-by: Junio C Hamano Signed-off-by: Johannes Schindelin --- builtin/gc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin/gc.c b/builtin/gc.c index 5e779662f2357f..0896c6f4758753 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -257,7 +257,7 @@ static const char *lock_repo_for_gc(int force, pid_t* ret_pid) int should_exit; if (!scan_fmt) - scan_fmt = xstrfmt("%s %%%dc", "%"SCNuMAX, HOST_NAME_MAX); + scan_fmt = xstrfmt("%s %%%ds", "%"SCNuMAX, HOST_NAME_MAX); fp = fopen(pidfile_path, "r"); memset(locking_host, 0, sizeof(locking_host)); should_exit = From 9a60ca312c103bd448400670af8814b9bfef8999 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 18 Dec 2017 15:25:15 +0100 Subject: [PATCH 15/16] msvc: make make_environment_block() thread-safe The buffer returned by _wenviron (which looks like a constant, but is not) is not thread-safe. This caused debug assertions when running t7814-grep-recurse-submodules with MSVC-built code (but only something like 9 out of 10 times). Let's use the thread-safe GetEnvironmentStringsW() function instead (which requires an accompanying call to FreeEnvironmentStringsW()). It is a little bit more expensive because it needs to lock and it needs to copy the entire environment block. But spawning a new process is *far* more expensive anyway. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 973965b2cda93a..e3a1546ecb2158 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1328,8 +1328,8 @@ static wchar_t *make_environment_block(char **deltaenv) * as a function that returns a pointer to a mostly static table. * Grab the pointer and cache it for the duration of our loop. */ - extern wchar_t **_wenviron; - const wchar_t **my_wenviron = _wenviron; + const wchar_t *my_wenviron = GetEnvironmentStringsW(); + const wchar_t *p; /* * Internally, we create normal 'C' arrays of strings (pointing @@ -1373,8 +1373,12 @@ static wchar_t *make_environment_block(char **deltaenv) nr_delta = nr_delta_ins + nr_delta_del; } - while (my_wenviron && my_wenviron[nr_wenv] && *my_wenviron[nr_wenv]) - maxlen += wcslen(my_wenviron[nr_wenv++]) + 1; + for (p = my_wenviron; p && *p; ) { + size_t len = wcslen(p) + 1; + maxlen += len; + p += len; + nr_wenv++; + } maxlen++; /* @@ -1439,11 +1443,13 @@ static wchar_t *make_environment_block(char **deltaenv) * ones into the result set. Note that we only have to de-dup WRT * the values from deltaenv, because the inherited set should be unique. */ + p = my_wenviron; for (j = 0; j < nr_wenv; j++) { - const wchar_t *v_j = my_wenviron[j]; + const wchar_t *v_j = p; wchar_t *v_j_eq = wcschr(v_j, L'='); int len_j_eq, len_j; + p += wcslen(p) + 1; if (!v_j_eq) continue; /* should not happen */ len_j_eq = v_j_eq + 1 - v_j; /* length(v_j) including '=' */ @@ -1469,6 +1475,7 @@ static wchar_t *make_environment_block(char **deltaenv) ; } + FreeEnvironmentStringsW(my_wenviron); if (wptrs_ins) free(wptrs_ins); if (wptrs_del) From 1a563edefaec7d379c57ef8abbfd6ce7e47266f6 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 18 Dec 2017 17:44:39 +0100 Subject: [PATCH 16/16] fixup! msvc: convert environment from/to UTF-16 on the fly It was discovered that the current make_environment_block() implementation was fixed incorrectly in eaae98ef59 (fixup! msvc: convert environment from/to UTF-16 on the fly, 2017-02-28): the total number of inserts was not adjusted when deltaenv tried to set a variable and then immediately delete it. This could result in an access violation, as we later looked through all keys slated to be inserted. This bug (and its age) is a pretty good indicator that the code may not have been all that obvious. So let's rewrite make_environment_block() from scratch. Let's replace it by a new implementation that 1) accumulates all environment variables into an array, 2) appends the deltaenv, 3) applies a stable sort, and finally 4) copies the entries (but only one for any given key) Signed-off-by: Johannes Schindelin --- compat/mingw.c | 235 +++++++++++++++++++++---------------------------- 1 file changed, 99 insertions(+), 136 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index e3a1546ecb2158..6ecf099a00c87e 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1311,6 +1311,33 @@ static char *path_lookup(const char *cmd, int exe_only) } #if defined(_MSC_VER) + +/* We need a stable sort */ +#ifndef INTERNAL_QSORT +#include "qsort.c" +#endif + +/* Compare only keys */ +static int wenvcmp(const void *a, const void *b) +{ + wchar_t *p = *(wchar_t **)a, *q = *(wchar_t **)b; + size_t p_len, q_len; + int ret; + + /* Find end of keys */ + for (p_len = 0; p[p_len] && p[p_len] != L'='; p_len++) + ; /* do nothing */ + for (q_len = 0; q[q_len] && q[q_len] != L'='; q_len++) + ; /* do nothing */ + + /* Are keys identical (modulo case)? */ + if (p_len == q_len && !_wcsnicmp(p, q, p_len)) + return 0; + + ret = _wcsnicmp(p, q, p_len < q_len ? p_len : q_len); + return ret ? ret : (p_len < q_len ? -1 : +1); +} + /* * Build an environment block combining the inherited environment * merged with the given list of settings. @@ -1318,6 +1345,8 @@ static char *path_lookup(const char *cmd, int exe_only) * Values of the form "KEY=VALUE" in deltaenv override inherited values. * Values of the form "KEY" in deltaenv delete inherited values. * + * Multiple entries in deltaenv for the same key are explicitly allowed. + * * We return a contiguous block of UNICODE strings with a final trailing * zero word. */ @@ -1328,162 +1357,96 @@ static wchar_t *make_environment_block(char **deltaenv) * as a function that returns a pointer to a mostly static table. * Grab the pointer and cache it for the duration of our loop. */ - const wchar_t *my_wenviron = GetEnvironmentStringsW(); - const wchar_t *p; + const wchar_t *wenv = GetEnvironmentStringsW(), *p; + size_t delta_size = 0, size = 1; /* for extra NUL at the end */ - /* - * Internally, we create normal 'C' arrays of strings (pointing - * into the blocks) to help with some of the de-dup work. - */ - wchar_t **wptrs_ins = NULL; - wchar_t **wptrs_del = NULL; - wchar_t *wblock_ins = NULL; - wchar_t *wblock_del = NULL; - wchar_t *wend_ins; - wchar_t *wend_del; - wchar_t *w_ins; - wchar_t *w_del; - - int maxlen = 0; - int nr_delta = 0; - int nr_delta_ins = 0; - int nr_delta_del = 0; - int nr_wenv = 0; - int j, k, k_ins, k_del; + wchar_t **array = NULL; + size_t alloc = 0, nr = 0, i; + + const char *p2; + wchar_t *wdeltaenv; + + wchar_t *result, *p3; /* - * Count the number of inserts and deletes in the deltaenv list. - * Allocate 'C' arrays for inserts and deletes. - * Also estimate the block size of our results. + * If there is no deltaenv to apply, simply return a copy */ - if (deltaenv && deltaenv[0] && *deltaenv[0]) { - for (k = 0; deltaenv && deltaenv[k] && *deltaenv[k]; k++) { - if (strchr(deltaenv[k], '=') == NULL) - nr_delta_del++; - else { - maxlen += strlen(deltaenv[k]) + 1; - nr_delta_ins++; - } + if (!deltaenv || !*deltaenv) { + for (p = wenv; p && *p; ) { + size_t s = wcslen(p) + 1; + size += s; + p += s; } - if (nr_delta_ins) - wptrs_ins = (wchar_t**)calloc(nr_delta_ins + 1, sizeof(wchar_t*)); - if (nr_delta_del) - wptrs_del = (wchar_t**)calloc(nr_delta_del + 1, sizeof(wchar_t*)); - - nr_delta = nr_delta_ins + nr_delta_del; - } - for (p = my_wenviron; p && *p; ) { - size_t len = wcslen(p) + 1; - maxlen += len; - p += len; - nr_wenv++; + ALLOC_ARRAY(result, size); + memcpy(result, wenv, size * sizeof(*wenv)); + FreeEnvironmentStringsW(wenv); + return result; } - maxlen++; /* - * Allocate blocks for inserted and deleted items. - * The individual pointers in the 'C' arrays will point into here. - * We will use the wblock_ins as the final result. + * If there is a deltaenv, let's accumulate all keys into `array`, + * sort them using the stable git_qsort() and then copy, skipping + * duplicate keys */ - if (nr_delta_del) { - wblock_del = (wchar_t*)calloc(maxlen, sizeof(wchar_t)); - wend_del = wblock_del + maxlen; - w_del = wblock_del; + + for (p = wenv; p && *p; ) { + size_t s = wcslen(p) + 1; + size += s; + ALLOC_GROW(array, nr + 1, alloc); + array[nr++] = p; + p += s; } - wblock_ins = (wchar_t*)calloc(maxlen, sizeof(wchar_t)); - wend_ins = wblock_ins + maxlen; - w_ins = wblock_ins; - /* - * deltaenv values override inherited environment, so put them - * in the result list first (so that we can de-dup using the - * wide versions of them. - * - * Items in the deltaenv list that DO NOT contain an "=" are - * treated as unsetenv. - * - * Care needs to be taken to handle entries that are added first, and - * then deleted. - */ - k_ins = 0; - k_del = 0; - for (k = 0; k < nr_delta; k++) { - if (strchr(deltaenv[k], '=') == NULL) { - wchar_t *save = w_del; - wptrs_del[k_del++] = w_del; - w_del += xutftowcs(w_del, deltaenv[k], (wend_del - w_del)); - *w_del++ = L'='; /* append '=' to make lookup easier in next step. */ - *w_del++ = 0; - - /* If we added this key, we have to remove it again */ - for (j = 0; j < k_ins; j++) - if (!wcsnicmp(wptrs_ins[j], save, w_del - save - 1)) { - if (j + 1 < k_ins) { - int delta = sizeof(wchar_t) * (wptrs_ins[j + 1] - wptrs_ins[j]), i; - memmove(wptrs_ins[j], wptrs_ins[j + 1], sizeof(wchar_t) * (w_ins - wptrs_ins[j + 1])); - for (i = j; i < --k_ins; i++) - wptrs_ins[i] = wptrs_ins[i + 1] - delta; - w_ins -= delta; - } else - w_ins = wptrs_ins[j]; - k_ins--; - j--; - } - } else { - wptrs_ins[k_ins++] = w_ins; - w_ins += xutftowcs(w_ins, deltaenv[k], (wend_ins - w_ins)) + 1; - } + /* (over-)assess size needed for wchar version of deltaenv */ + for (i = 0; deltaenv[i]; i++) { + size_t s = strlen(deltaenv[i]) + 1; + delta_size += s; } - assert(k_ins <= nr_delta_ins); - assert(k_del == nr_delta_del); - /* - * Walk the inherited environment and copy over unique, non-deleted - * ones into the result set. Note that we only have to de-dup WRT - * the values from deltaenv, because the inherited set should be unique. - */ - p = my_wenviron; - for (j = 0; j < nr_wenv; j++) { - const wchar_t *v_j = p; - wchar_t *v_j_eq = wcschr(v_j, L'='); - int len_j_eq, len_j; - - p += wcslen(p) + 1; - if (!v_j_eq) - continue; /* should not happen */ - len_j_eq = v_j_eq + 1 - v_j; /* length(v_j) including '=' */ - - /* lookup v_j in list of to-delete vars */ - for (k_del = 0; k_del < nr_delta_del; k_del++) { - if (wcsnicmp(v_j, wptrs_del[k_del], len_j_eq) == 0) - goto skip_it; - } + ALLOC_ARRAY(wdeltaenv, delta_size); - /* lookup v_j in deltaenv portion of result set */ - for (k_ins = 0; k_ins < nr_delta_ins; k_ins++) { - if (wcsnicmp(v_j, wptrs_ins[k_ins], len_j_eq) == 0) - goto skip_it; - } + /* convert the deltaenv, appending to array */ + for (i = 0, p3 = wdeltaenv; deltaenv[i]; i++) { + size_t s = strlen(deltaenv[i]) + 1, wlen; + wlen = xutftowcs(p3, deltaenv[i], s * 2); - /* item is unique, add it to results. */ - len_j = wcslen(v_j); - memcpy(w_ins, v_j, len_j * sizeof(wchar_t)); - w_ins += len_j + 1; + ALLOC_GROW(array, nr + 1, alloc); + array[nr++] = p3; -skip_it: - ; + p3 += wlen + 1; } - FreeEnvironmentStringsW(my_wenviron); - if (wptrs_ins) - free(wptrs_ins); - if (wptrs_del) - free(wptrs_del); - if (wblock_del) - free(wblock_del); + git_qsort(array, nr, sizeof(*array), wenvcmp); + ALLOC_ARRAY(result, size + delta_size); + + for (p3 = result, i = 0; i < nr; i++) { + wchar_t *equal = wcschr(array[i], L'=');; - return wblock_ins; + /* Skip "to delete" entry */ + if (!equal) + continue; + + p = array[i]; + + /* Skip any duplicate */ + if (i + 1 < nr) { + wchar_t *next = array[i + 1]; + size_t n = equal - p; + + if (!_wcsnicmp(p, next, n) && (!next[n] || next[n] == L'=')) + continue; + } + + size = wcslen(p) + 1; + memcpy(p3, p, size * sizeof(*p)); + p3 += size; + } + *p3 = L'\0'; + + free(array); + FreeEnvironmentStringsW(wenv); + return result; } #else