Make nix-shells functionnal. Run tests on hydra.

IntersectMBO · Feb 6, 2020 · 7bc4f40 · 7bc4f40
1 parent a1572fe
commit 7bc4f40
Show file tree

Hide file tree

Showing 9 changed files with 155 additions and 119 deletions.
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -1,21 +1,22 @@
 steps:
-  #- label: 'stack rebuild'
-  #  env:
-  #    STACK_ROOT: "/build/cardano-node.stack"
-  #  command:
-  #    - "rm -rf /build/cardano-node"
-  #    - "cp -R . /build/cardano-node"
-  #    - "cd /build/cardano-node"
-  #    - "nix-build scripts/buildkite -o stack-rebuild"
-  #    - "./stack-rebuild"
-  #  agents:
-  #    system: x86_64-linux
+  - label: 'stack rebuild'
+    env:
+      STACK_ROOT: "/build/cardano-node.stack"
+    command:
+      - "rm -rf /build/cardano-node"
+      - "cp -R . /build/cardano-node"
+      - "cd /build/cardano-node"
+      - "nix-build scripts/buildkite -o stack-rebuild"
+      - "./stack-rebuild"
+    agents:
+      system: x86_64-linux
 
   - label: 'check-cabal-project'
     command: 'nix-build lib.nix -A iohkNix.checkCabalProject -o check-cabal-project.sh && ./check-cabal-project.sh'
     agents:
       system: x86_64-linux
 
+# FIXME, waiting for https://github.com/input-output-hk/haskell.nix/pull/426
 #  - label: 'release.nix'
 #    command: 'nix-build -A check-hydra -o check-hydra.sh && ./check-hydra.sh'
 #    agents:

diff --git a/cardano-config/cardano-config.cabal b/cardano-config/cardano-config.cabal
@@ -20,7 +20,7 @@ library
                        Cardano.Config.Topology
                        Cardano.Config.Types
 
-  build-depends:       base >=4.12 && <4.13
+  build-depends:       base >=4.12 && <5
                      , aeson
                      , async
                      , bytestring

diff --git a/default.nix b/default.nix
@@ -1,6 +1,7 @@
 { system ? builtins.currentSystem
 , crossSystem ? null
 , config ? {}
+, sourcesOverride ? {}
 , profiling ? false
 , commonLib ? import ./lib.nix { inherit system crossSystem config profiling; }
 , pkgs ? commonLib.pkgs
@@ -12,7 +13,7 @@
 
 let
   lib = commonLib.pkgs.lib;
-  inherit (commonLib) environments haskellPackages;
+  inherit (commonLib) environments haskellPackages niv;
   cardano-node = haskellPackages.cardano-node.components.exes.cardano-node;
 
   scripts = commonLib.pkgs.callPackage ./nix/scripts.nix {
@@ -25,7 +26,7 @@ let
   };
 
   # we are only intersted in listing the project packages
-  projectHaskellPackages = pkgs.haskell-nix.haskellLib.selectProjectPackages haskellPackages;
+  projectHaskellPackages = commonLib.selectProjectPackages haskellPackages;
 
   self = with commonLib; {
     inherit scripts nixosTests environments cardano-node;
@@ -34,34 +35,46 @@ let
     inherit (iohkNix) check-hydra;
 
     # `tests` are the test suites which have been built.
-    tests = collectComponents "tests" isCardanoNode haskellPackages;
-    # `checks` are the result of executing the tests.
-    checks = pkgs.recurseIntoAttrs (getPackageChecks (filterCardanoPackages haskellPackages));
-    # `benchmarks` are only built, not run.
-    benchmarks = collectComponents "benchmarks" isCardanoNode haskellPackages;
+    tests = collectComponents' "tests" projectHaskellPackages;
+    # `benchmarks` (only built, not run).
+    benchmarks = collectComponents' "benchmarks" projectHaskellPackages;
+    # `checks` collect results of executing the benchmarks and tests:
+    checks = {
+      benchmarks = collectChecks self.benchmarks;
+      tests = collectChecks self.tests;
+    } // { recurseForDerivations = true; };
 
     shell = haskellPackages.shellFor {
 
-      #packages = ps: with ps; [
-      #  haskellPackages.cardano-node
-      #  haskellPackages.cardano-config
-      #];
+      packages = ps: with ps; [
+        ps.cardano-node
+        ps.cardano-config
+        # in theory we should only have the above two packages (or better, they should be auto-detected),
+        # but due to source-repository-package declarations being considered as local packages by cabal, we need the following packages as well.
+        # cf. https://github.com/haskell/cabal/issues/6249 and https://github.com/haskell/cabal/issues/5444
+        ps.cardano-sl-x509
+        ps.ekg-prometheus-adapter
+        ps.ouroboros-consensus
+        ps.ouroboros-network
+      ];
 
       # Builds a Hoogle documentation index of all dependencies,
       # and provides a "hoogle" command to search the index.
       inherit withHoogle;
 
       # You might want some extra tools in the shell (optional).
-      buildInputs = (with haskellPackages; [
-        #weeder.components.exes.weeder
-        #hlint.components.exes.hlint
-        #cabal-install.components.exes.cabal
-        #ghcid.components.exes.ghcid
-      ]) ++ (with pkgs; [
+      buildInputs = with pkgs; [
+        cabal-install
+        ghcid
+        hlint
+        pkgs.haskellPackages.weeder
+        nix
+        niv
         pkgconfig
         sqlite-interactive
         tmux
-      ]);
+        git
+      ];
 
       # Prevents cabal from choosing alternate plans, so that
       # *all* dependencies are provided by Nix.

diff --git a/lib.nix b/lib.nix
@@ -2,46 +2,41 @@
 , crossSystem ? null
 , config ? {}
 , overlays ? []
+, sourcesOverride ? {}
 , profiling ? false
 }:
 
 let
-  sources = import ./nix/sources.nix;
+  # use default stable nixpkgs from iohk-nix instead of our own:
+  sources = removeAttrs (import ./nix/sources.nix) [ "nixpkgs" ] // sourcesOverride;
   iohkNix = import sources.iohk-nix {
+    inherit config system crossSystem;
     sourcesOverride = sources;
+    nixpkgsOverlays = overlays;
   };
-  haskellNix = import sources."haskell.nix";
-  args = haskellNix // {
-    inherit system crossSystem;
-    overlays = (haskellNix.overlays or []) ++ overlays;
-    config = (haskellNix.config or {}) // config;
-  };
-  nixpkgs = import sources.nixpkgs;
-  pkgs = nixpkgs args;
+  pkgs = iohkNix.pkgs;
   haskellPackages = import ./nix/pkgs.nix {
     inherit pkgs profiling;
     src = ./.;
   };
   # TODO: add haskellPackages and fix svclib
   svcLib = import ./nix/svclib.nix { inherit pkgs; cardano-node-packages = haskellPackages.cardano-node.components.exes; };
   lib = pkgs.lib;
-  niv = (import sources.niv {}).niv;
-  isCardanoNode = with lib; package:
-    (package.isHaskell or false) &&
-      ((hasPrefix "cardano-node" package.identifier.name) ||
-       (elem package.identifier.name [ "text-class" "bech32" ]));
-  filterCardanoPackages = pkgs.lib.filterAttrs (_: package: isCardanoNode package);
-  getPackageChecks = pkgs.lib.mapAttrs (_: package: package.checks);
+
+  collectChecks = lib.mapAttrsRecursiveCond (p: !(lib.isDerivation p))
+    (_: p: if (lib.isAttrs p) then pkgs.haskell-nix.haskellLib.check p else p);
+
+  collectComponents' = group: pkgs.haskell-nix.haskellLib.collectComponents group (_:true);
+
 in lib // iohkNix.cardanoLib // {
-  inherit (pkgs.haskell-nix.haskellLib) collectComponents;
+  inherit (pkgs.haskell-nix.haskellLib) collectComponents selectProjectPackages;
+  inherit (iohkNix) niv;
   inherit
-    niv
     sources
     haskellPackages
     pkgs
     iohkNix
     svcLib
-    isCardanoNode
-    getPackageChecks
-    filterCardanoPackages;
+    collectChecks
+    collectComponents';
 }
diff --git a/nix/pkgs.nix b/nix/pkgs.nix
@@ -20,63 +20,63 @@ let
         hasPrefix (toString origSrc + toString dir) path;
     } + dir;
 
-  recRecurseIntoAttrs = with pkgs; pred: x: if pred x then recurseIntoAttrs (lib.mapAttrs (n: v: if n == "buildPackages" then v else recRecurseIntoAttrs pred v) x) else x;
-  pkgSet = recRecurseIntoAttrs (x: with pkgs; lib.isAttrs x && !lib.isDerivation x)
-    # we are only intersted in listing the project packages
-    (pkgs.lib.filterAttrs (with pkgs.haskell-nix.haskellLib; (n: p: p != null && (isLocalPackage p && isProjectPackage p) || n == "shellFor"))
-      # from our project which is based on a cabal project.
-      (pkgs.haskell-nix.cabalProject {
-          src = pkgs.haskell-nix.haskellLib.cleanGit { inherit src; };
-          ghc = pkgs.buildPackages.haskell-nix.compiler.${haskellCompiler};
-          modules = [
-            # Allow reinstallation of Win32
-            { nonReinstallablePkgs =
-              [ "rts" "ghc-heap" "ghc-prim" "integer-gmp" "integer-simple" "base"
-                "deepseq" "array" "ghc-boot-th" "pretty" "template-haskell"
-                # ghcjs custom packages
-                "ghcjs-prim" "ghcjs-th"
-                "ghc-boot"
-                "ghc" "array" "binary" "bytestring" "containers"
-                "filepath" "ghc-boot" "ghc-compact" "ghc-prim"
-                # "ghci" "haskeline"
-                "hpc"
-                "mtl" "parsec" "text" "transformers"
-                "xhtml"
-                # "stm" "terminfo"
-              ];
-            }
-            {
-              # Packages we wish to ignore version bounds of.
-              # This is similar to jailbreakCabal, however it
-              # does not require any messing with cabal files.
-              #doCheck = false;
-              packages.katip.doExactConfig = true;
-              packages.ekg.components.library.enableSeparateDataOutput = true;
-              packages.cardano-node.configureFlags = [ "--ghc-option=-Werror" ];
-              packages.cardano-node.doCheck = true;
-              packages.cardano-config.configureFlags = [ "--ghc-option=-Werror" ];
-              packages.cardano-config.doCheck = true;
-              enableLibraryProfiling = profiling;
-           }
-           (pkgs.lib.optionalAttrs pkgs.stdenv.hostPlatform.isWindows {
-             # Disable cabal-doctest tests by turning off custom setups
-             packages.comonad.package.buildType = pkgs.lib.mkForce "Simple";
-             packages.distributive.package.buildType = pkgs.lib.mkForce "Simple";
-             packages.lens.package.buildType = pkgs.lib.mkForce "Simple";
-             packages.nonempty-vector.package.buildType = pkgs.lib.mkForce "Simple";
-             packages.semigroupoids.package.buildType = pkgs.lib.mkForce "Simple";
+  pkg-set = pkgs.haskell-nix.cabalProject {
+    src = pkgs.haskell-nix.haskellLib.cleanGit { inherit src; };
+    ghc = pkgs.buildPackages.haskell-nix.compiler.${haskellCompiler};
+    modules = [
 
-             # Make sure we use a buildPackages version of happy
-             packages.pretty-show.components.library.build-tools = [ pkgs.buildPackages.haskell-nix.haskellPackages.happy ];
+      # Allow reinstallation of Win32
+      { nonReinstallablePkgs =
+        [ "rts" "ghc-heap" "ghc-prim" "integer-gmp" "integer-simple" "base"
+          "deepseq" "array" "ghc-boot-th" "pretty" "template-haskell"
+          # ghcjs custom packages
+          "ghcjs-prim" "ghcjs-th"
+          "ghc-boot"
+          "ghc" "array" "binary" "bytestring" "containers"
+          "filepath" "ghc-boot" "ghc-compact" "ghc-prim"
+          # "ghci" "haskeline"
+          "hpc"
+          "mtl" "parsec" "text" "transformers"
+          "xhtml"
+          # "stm" "terminfo"
+        ];
+      }
+      {
+        # Packages we wish to ignore version bounds of.
+        # This is similar to jailbreakCabal, however it
+        # does not require any messing with cabal files.
+        packages.katip.doExactConfig = true;
 
-             # Remove hsc2hs build-tool dependencies (suitable version will be available as part of the ghc derivation)
-             packages.Win32.components.library.build-tools = pkgs.lib.mkForce [];
-             packages.terminal-size.components.library.build-tools = pkgs.lib.mkForce [];
-             packages.network.components.library.build-tools = pkgs.lib.mkForce [];
-           })
-          ];
-        # TODO add flags to packages (like cs-ledger) so we can turn off tests that will
-        # not build for windows on a per package bases (rather than using --disable-tests).
-        configureArgs = pkgs.lib.optionalString pkgs.stdenv.hostPlatform.isWindows "--disable-tests";
-      }));
- in pkgSet
+        # split data output for ekg to reduce closure size
+        packages.ekg.components.library.enableSeparateDataOutput = true;
+        packages.cardano-node.configureFlags = [ "--ghc-option=-Werror" ];
+        packages.cardano-config.configureFlags = [ "--ghc-option=-Werror" ];
+        enableLibraryProfiling = profiling;
+
+        # some packages are missing identifier.name:
+        packages.Win32.package.identifier.name = "Win32";
+        packages.file-embed-lzma.package.identifier.name = "file-embed-lzma";
+        packages.singletons.package.identifier.name = "singletons";
+      }
+      (pkgs.lib.optionalAttrs pkgs.stdenv.hostPlatform.isWindows {
+        # Disable cabal-doctest tests by turning off custom setups
+        packages.comonad.package.buildType = pkgs.lib.mkForce "Simple";
+        packages.distributive.package.buildType = pkgs.lib.mkForce "Simple";
+        packages.lens.package.buildType = pkgs.lib.mkForce "Simple";
+        packages.nonempty-vector.package.buildType = pkgs.lib.mkForce "Simple";
+        packages.semigroupoids.package.buildType = pkgs.lib.mkForce "Simple";
+
+        # Make sure we use a buildPackages version of happy
+        packages.pretty-show.components.library.build-tools = [ pkgs.buildPackages.haskell-nix.haskellPackages.happy ];
+
+        # Remove hsc2hs build-tool dependencies (suitable version will be available as part of the ghc derivation)
+        packages.Win32.components.library.build-tools = pkgs.lib.mkForce [];
+        packages.terminal-size.components.library.build-tools = pkgs.lib.mkForce [];
+        packages.network.components.library.build-tools = pkgs.lib.mkForce [];
+      })
+    ];
+    # TODO add flags to packages (like cs-ledger) so we can turn off tests that will
+    # not build for windows on a per package bases (rather than using --disable-tests).
+    configureArgs = pkgs.lib.optionalString pkgs.stdenv.hostPlatform.isWindows "--disable-tests";
+  };
+in pkg-set
diff --git a/nix/sources.json b/nix/sources.json
@@ -60,15 +60,15 @@
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "iohk-nix": {
-        "branch": "master",
+        "branch": "update-nixpkgs-haskell-nix",
         "description": "nix scripts shared across projects",
         "homepage": null,
         "owner": "input-output-hk",
         "repo": "iohk-nix",
-        "rev": "40fc58f0f1cb0f9de24d8df3144a8cd8f2e3a775",
-        "sha256": "1h3d1gmjfxbwrdszj1ciqlbn646fy687w940fs4gx2bc12j39acb",
+        "rev": "af3f56bb76effb6462b8f56db46b09fa5134f412",
+        "sha256": "12dsqs31ij5i3bn0cq9s4cazcf9rd70rs9hrlk05fdylz1i2swm1",
         "type": "tarball",
-        "url": "https://github.com/input-output-hk/iohk-nix/archive/40fc58f0f1cb0f9de24d8df3144a8cd8f2e3a775.tar.gz",
+        "url": "https://github.com/input-output-hk/iohk-nix/archive/af3f56bb76effb6462b8f56db46b09fa5134f412.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "niv": {

diff --git a/nix/stack-shell.nix b/nix/stack-shell.nix
@@ -0,0 +1,8 @@
+
+with import ../lib.nix {};
+
+pkgs.haskell.lib.buildStackProject {
+  name = "cardano-node-stack-env";
+  buildInputs = with pkgs; [ zlib openssl gmp libffi git systemd haskellPackages.happy ];
+  ghc = (import ../shell.nix {}).ghc;
+}
diff --git a/release.nix b/release.nix
@@ -104,7 +104,7 @@ let
     # TODO: fix broken evals
     #musl64 = mapTestOnCross musl64 (packagePlatformsCross project);
   } // extraBuilds // (mkRequiredJob (
-      collectTests jobs.native.tests ++
+      collectTests jobs.native.checks.tests ++
       collectTests jobs.native.benchmarks ++ [
       jobs.native.cardano-node.x86_64-darwin
       jobs.native.cardano-node.x86_64-linux