From 00b2b698b73eb881a6b298a6fc6047512b7bba13 Mon Sep 17 00:00:00 2001 From: Hamish Mackenzie Date: Thu, 17 Mar 2022 19:12:17 +1300 Subject: [PATCH 1/5] Make DLLs in deps available to wine TH runner This is a better fix to the problem of making DLLs available to the process running in wine that is used to evaluate TH code when cross compiling for Windows. --- builder/comp-builder.nix | 32 ++++++++++++++++++++------------ overlays/mingw_w64.nix | 28 ++++++++++++++++++++++++++-- overlays/windows.nix | 2 +- test/default.nix | 1 + test/th-dlls/default.nix | 30 ++++++++++++++++++++++++++++++ test/th-dlls/src/Lib.hs | 11 +++++++++++ test/th-dlls/th-dlls.cabal | 16 ++++++++++++++++ 7 files changed, 105 insertions(+), 15 deletions(-) create mode 100644 test/th-dlls/default.nix create mode 100644 test/th-dlls/src/Lib.hs create mode 100644 test/th-dlls/th-dlls.cabal diff --git a/builder/comp-builder.nix b/builder/comp-builder.nix index b2795ad2ac..cfeb6a871c 100644 --- a/builder/comp-builder.nix +++ b/builder/comp-builder.nix @@ -382,18 +382,26 @@ let runHook postConfigure ''; - buildPhase = if stdenv.hostPlatform.isGhcjs then '' - runHook preBuild - # https://gitlab.haskell.org/ghc/ghc/issues/9221 - $SETUP_HS build ${haskellLib.componentTarget componentId} ${lib.concatStringsSep " " setupBuildFlags} - runHook postBuild - '' else '' - runHook preBuild - # https://gitlab.haskell.org/ghc/ghc/issues/9221 - $SETUP_HS build ${haskellLib.componentTarget componentId} -j$(($NIX_BUILD_CORES > 4 ? 4 : $NIX_BUILD_CORES)) ${lib.concatStringsSep " " setupBuildFlags} - runHook postBuild - '' - ; + buildPhase = + # It seems that by the time the iserv wrapper specifiec by `--ghc-option=-pgmi` runs + # all the array environment variables are removed from the environment. To get a list + # of all the locations a DLLs might be present we need access to pkgsHostTarget. + # Adding a string version of the list array of nix store paths allows us to get that + # list when we need it. + (lib.optionalString stdenv.hostPlatform.isWindows '' + export pkgsHostTargetAsString="''${pkgsHostTarget[@]}" + '') + + (if stdenv.hostPlatform.isGhcjs then '' + runHook preBuild + # https://gitlab.haskell.org/ghc/ghc/issues/9221 + $SETUP_HS build ${haskellLib.componentTarget componentId} ${lib.concatStringsSep " " setupBuildFlags} + runHook postBuild + '' else '' + runHook preBuild + # https://gitlab.haskell.org/ghc/ghc/issues/9221 + $SETUP_HS build ${haskellLib.componentTarget componentId} -j$(($NIX_BUILD_CORES > 4 ? 4 : $NIX_BUILD_CORES)) ${lib.concatStringsSep " " setupBuildFlags} + runHook postBuild + ''); # Note: Cabal does *not* copy test executables during the `install` phase. # diff --git a/overlays/mingw_w64.nix b/overlays/mingw_w64.nix index 746b801b11..a68c7e610f 100644 --- a/overlays/mingw_w64.nix +++ b/overlays/mingw_w64.nix @@ -28,7 +28,19 @@ let unset configureFlags PORT=$((5000 + $RANDOM % 5000)) (>&2 echo "---> Starting remote-iserv on port $PORT") - WINEDLLOVERRIDES="winemac.drv=d" WINEDEBUG=warn-all,fixme-all,-menubuilder,-mscoree,-ole,-secur32,-winediag WINEPREFIX=$TMP ${wine}/bin/wine64 ${remote-iserv}/bin/remote-iserv.exe tmp $PORT & + REMOTE_ISERV=$(mktemp -d) + ln -s ${remote-iserv}/bin/* $REMOTE_ISERV + for p in $pkgsHostTargetAsString; do + find "$p" -iname '*.dll' -exec ln -s {} $REMOTE_ISERV \; + find "$p" -iname '*.dll.a' -exec ln -s {} $REMOTE_ISERV \; + done + ( + cd $REMOTE_ISERV + for l in lib*.dll; do + if [[ ! -e "''${l#lib}" ]]; then ln -s "$l" "''${l#lib}"; fi + done + ) + WINEDLLOVERRIDES="winemac.drv=d" WINEDEBUG=warn-all,fixme-all,-menubuilder,-mscoree,-ole,-secur32,-winediag WINEPREFIX=$TMP ${wine}/bin/wine64 $REMOTE_ISERV/remote-iserv.exe tmp $PORT & (>&2 echo "---| remote-iserv should have started on $PORT") RISERV_PID="$!" ${iserv-proxy}/bin/iserv-proxy $@ 127.0.0.1 "$PORT" @@ -46,7 +58,19 @@ let unset configureFlags PORT=$((5000 + $RANDOM % 5000)) (>&2 echo "---> Starting remote-iserv on port $PORT") - WINEDLLOVERRIDES="winemac.drv=d" WINEDEBUG=warn-all,fixme-all,-menubuilder,-mscoree,-ole,-secur32,-winediag WINEPREFIX=$TMP ${wine}/bin/wine64 ${remote-iserv.override { enableProfiling = true; }}/bin/remote-iserv.exe tmp $PORT & + REMOTE_ISERV=$(mktemp -d) + ln -s ${remote-iserv.override { enableProfiling = true; }}/bin/* $REMOTE_ISERV + for p in $pkgsHostTargetAsString; do + find "$p" -iname '*.dll' -exec ln -s {} $REMOTE_ISERV \; + find "$p" -iname '*.dll.a' -exec ln -s {} $REMOTE_ISERV \; + done + ( + cd $REMOTE_ISERV + for l in lib*.dll; do + if [[ ! -e "''${l#lib}" ]]; then ln -s "$l" "''${l#lib}"; fi + done + ) + WINEDLLOVERRIDES="winemac.drv=d" WINEDEBUG=warn-all,fixme-all,-menubuilder,-mscoree,-ole,-secur32,-winediag WINEPREFIX=$TMP ${wine}/bin/wine64 $REMOTE_ISERV/remote-iserv.exe tmp $PORT & (>&2 echo "---| remote-iserv should have started on $PORT") RISERV_PID="$!" ${iserv-proxy}/bin/iserv-proxy $@ 127.0.0.1 "$PORT" diff --git a/overlays/windows.nix b/overlays/windows.nix index 610b1cc7c4..62aa3534ca 100644 --- a/overlays/windows.nix +++ b/overlays/windows.nix @@ -73,7 +73,7 @@ final: prev: # dependencies) and then placing them somewhere where wine+remote-iserv # will find them. remote-iserv.postInstall = pkgs.lib.optionalString pkgs.stdenv.hostPlatform.isWindows ( - let extra-libs = [ pkgs.openssl.bin pkgs.libffi pkgs.gmp (pkgs.libsodium-vrf or pkgs.libsodium) pkgs.windows.mcfgthreads pkgs.buildPackages.gcc.cc pkgs.secp256k1 ]; in '' + let extra-libs = [ pkgs.libffi pkgs.gmp pkgs.windows.mcfgthreads pkgs.buildPackages.gcc.cc ]; in '' for p in ${lib.concatStringsSep " "extra-libs}; do find "$p" -iname '*.dll' -exec cp {} $out/bin/ \; find "$p" -iname '*.dll.a' -exec cp {} $out/bin/ \; diff --git a/test/default.nix b/test/default.nix index 31e77b8d4f..3242e89a9e 100644 --- a/test/default.nix +++ b/test/default.nix @@ -202,6 +202,7 @@ let sublib-docs = callTest ./sublib-docs { inherit util compiler-nix-name; }; githash = haskell-nix.callPackage ./githash { inherit compiler-nix-name; testSrc = testSrcWithGitDir; }; c-ffi = callTest ./c-ffi { inherit util compiler-nix-name; }; + th-dlls = callTest ./th-dlls { inherit util compiler-nix-name; }; unit = unitTests; }; diff --git a/test/th-dlls/default.nix b/test/th-dlls/default.nix new file mode 100644 index 0000000000..a53ed047d9 --- /dev/null +++ b/test/th-dlls/default.nix @@ -0,0 +1,30 @@ +# Test a package set +{ stdenv, lib, util, mkCabalProjectPkgSet, project', haskellLib, recurseIntoAttrs, testSrc, compiler-nix-name }: + +with lib; + +let + modules = [ + { + # Package has no exposed modules which causes + # haddock: No input file(s) + packages.cabal-simple.doHaddock = false; + } + ]; + + project = project' { + inherit compiler-nix-name; + src = testSrc "th-dlls"; + inherit modules; + }; + + packages = project.hsPkgs; + +in recurseIntoAttrs { + ifdInputs = { + inherit (project) plan-nix; + }; + + build = packages.th-dlls.components.library; + build-profiled = packages.th-dlls.components.library.profiled; +} diff --git a/test/th-dlls/src/Lib.hs b/test/th-dlls/src/Lib.hs new file mode 100644 index 0000000000..64c3891c0e --- /dev/null +++ b/test/th-dlls/src/Lib.hs @@ -0,0 +1,11 @@ +{-# LANGUAGE TemplateHaskell #-} +module Lib where + +import Control.Monad.IO.Class (liftIO) +import OpenSSL (withOpenSSL) +import OpenSSL.BN (withBN) +import Libsodium (sodium_init) +import Language.Haskell.TH.Syntax (Exp(..), Lit(..)) + +x = $(liftIO (withOpenSSL (withBN 0 (\_ -> return (LitE (IntegerL 0)))))) +y = $(liftIO (sodium_init >> return (LitE (IntegerL 0)))) diff --git a/test/th-dlls/th-dlls.cabal b/test/th-dlls/th-dlls.cabal new file mode 100644 index 0000000000..b71c2aa0e2 --- /dev/null +++ b/test/th-dlls/th-dlls.cabal @@ -0,0 +1,16 @@ +cabal-version: >=1.10 +name: th-dlls +version: 0.1.0.0 +license: PublicDomain +author: Hamish Mackenzie +maintainer: Hamish.K.Mackenzie@gmail.com +build-type: Simple + +library + build-depends: base + , HsOpenSSL + , libsodium + , template-haskell + exposed-modules: Lib + hs-source-dirs: src + default-language: Haskell2010 From b093f93291a35fb05005b20a8bf07ec459cca0e4 Mon Sep 17 00:00:00 2001 From: Hamish Mackenzie Date: Thu, 17 Mar 2022 21:29:47 +1300 Subject: [PATCH 2/5] Switch off th-dlls test for ghcjs --- test/th-dlls/default.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/th-dlls/default.nix b/test/th-dlls/default.nix index a53ed047d9..8dfacb1f2a 100644 --- a/test/th-dlls/default.nix +++ b/test/th-dlls/default.nix @@ -21,6 +21,8 @@ let packages = project.hsPkgs; in recurseIntoAttrs { + meta.disabled = stdenv.hostPlatform.isGhcjs; + ifdInputs = { inherit (project) plan-nix; }; From 0c24959eecc8d440c3af70e9486a010c150b342a Mon Sep 17 00:00:00 2001 From: Hamish Mackenzie Date: Thu, 17 Mar 2022 21:49:06 +1300 Subject: [PATCH 3/5] Clean up --- test/th-dlls/default.nix | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/test/th-dlls/default.nix b/test/th-dlls/default.nix index 8dfacb1f2a..3dbf595a2b 100644 --- a/test/th-dlls/default.nix +++ b/test/th-dlls/default.nix @@ -1,21 +1,12 @@ -# Test a package set -{ stdenv, lib, util, mkCabalProjectPkgSet, project', haskellLib, recurseIntoAttrs, testSrc, compiler-nix-name }: +# Test building TH code that needs DLLs when cross compiling for windows +{ stdenv, lib, project', haskellLib, recurseIntoAttrs, testSrc, compiler-nix-name }: with lib; let - modules = [ - { - # Package has no exposed modules which causes - # haddock: No input file(s) - packages.cabal-simple.doHaddock = false; - } - ]; - project = project' { inherit compiler-nix-name; src = testSrc "th-dlls"; - inherit modules; }; packages = project.hsPkgs; From eff95c9ef2772a7e39523edce3f41acf3a82e03c Mon Sep 17 00:00:00 2001 From: Hamish Mackenzie Date: Thu, 17 Mar 2022 22:00:28 +1300 Subject: [PATCH 4/5] Clean up --- test/th-dlls/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/th-dlls/default.nix b/test/th-dlls/default.nix index 3dbf595a2b..cca2d3ecec 100644 --- a/test/th-dlls/default.nix +++ b/test/th-dlls/default.nix @@ -1,5 +1,5 @@ # Test building TH code that needs DLLs when cross compiling for windows -{ stdenv, lib, project', haskellLib, recurseIntoAttrs, testSrc, compiler-nix-name }: +{ stdenv, lib, util, project', haskellLib, recurseIntoAttrs, testSrc, compiler-nix-name }: with lib; From 3567478edb5e2a84cde164f14bc363037fbd4a13 Mon Sep 17 00:00:00 2001 From: Hamish Mackenzie Date: Fri, 18 Mar 2022 13:32:42 +1300 Subject: [PATCH 5/5] Deduplication --- overlays/mingw_w64.nix | 36 +++--------------------------------- 1 file changed, 3 insertions(+), 33 deletions(-) diff --git a/overlays/mingw_w64.nix b/overlays/mingw_w64.nix index a68c7e610f..f67bbff446 100644 --- a/overlays/mingw_w64.nix +++ b/overlays/mingw_w64.nix @@ -18,7 +18,7 @@ let configureFlags = lib.optional hostPlatform.isWindows "--disable-split-sections"; - wineIservWrapperVanilla = writeScriptBin "iserv-wrapper" '' + wineIservWrapperScript = enableProfiling: writeScriptBin ("iserv-wrapper" + lib.optionalString enableProfiling "-prof") '' #!${stdenv.shell} set -euo pipefail # unset the configureFlags. @@ -29,7 +29,7 @@ let PORT=$((5000 + $RANDOM % 5000)) (>&2 echo "---> Starting remote-iserv on port $PORT") REMOTE_ISERV=$(mktemp -d) - ln -s ${remote-iserv}/bin/* $REMOTE_ISERV + ln -s ${remote-iserv.override { inherit enableProfiling; }}/bin/* $REMOTE_ISERV for p in $pkgsHostTargetAsString; do find "$p" -iname '*.dll' -exec ln -s {} $REMOTE_ISERV \; find "$p" -iname '*.dll.a' -exec ln -s {} $REMOTE_ISERV \; @@ -48,37 +48,7 @@ let kill $RISERV_PID ''; - wineIservWrapperProf = writeScriptBin "iserv-wrapper-prof" '' - #!${stdenv.shell} - set -euo pipefail - # unset the configureFlags. - # configure should have run already - # without restting it, wine might fail - # due to a too large environment. - unset configureFlags - PORT=$((5000 + $RANDOM % 5000)) - (>&2 echo "---> Starting remote-iserv on port $PORT") - REMOTE_ISERV=$(mktemp -d) - ln -s ${remote-iserv.override { enableProfiling = true; }}/bin/* $REMOTE_ISERV - for p in $pkgsHostTargetAsString; do - find "$p" -iname '*.dll' -exec ln -s {} $REMOTE_ISERV \; - find "$p" -iname '*.dll.a' -exec ln -s {} $REMOTE_ISERV \; - done - ( - cd $REMOTE_ISERV - for l in lib*.dll; do - if [[ ! -e "''${l#lib}" ]]; then ln -s "$l" "''${l#lib}"; fi - done - ) - WINEDLLOVERRIDES="winemac.drv=d" WINEDEBUG=warn-all,fixme-all,-menubuilder,-mscoree,-ole,-secur32,-winediag WINEPREFIX=$TMP ${wine}/bin/wine64 $REMOTE_ISERV/remote-iserv.exe tmp $PORT & - (>&2 echo "---| remote-iserv should have started on $PORT") - RISERV_PID="$!" - ${iserv-proxy}/bin/iserv-proxy $@ 127.0.0.1 "$PORT" - (>&2 echo "---> killing remote-iserv...") - kill $RISERV_PID - ''; - - wineIservWrapper = symlinkJoin { name = "iserv-wrapper"; paths = [ wineIservWrapperVanilla wineIservWrapperProf ]; }; + wineIservWrapper = symlinkJoin { name = "iserv-wrapper"; paths = [ (wineIservWrapperScript false) (wineIservWrapperScript true) ]; }; ################################################################################ # Build logic (TH support via remote iserv via wine)