From 911e0a86af4801d19f5ce34112e5d3fd4d949c2a Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Giraudeau Date: Fri, 7 Feb 2020 00:06:19 +0100 Subject: [PATCH] Migrate to overlay style. --- .buildkite/pipeline.yml | 2 +- benchmarking/chain-sync/ci.sh | 2 +- default.nix | 97 ++++------ lib.nix | 42 ----- nix/default.nix | 29 +++ nix/haskell.nix | 77 ++++++++ nix/nixos/cardano-cluster-service.nix | 2 +- nix/nixos/cardano-node-service.nix | 6 +- nix/nixos/chairman-as-a-service.nix | 6 +- nix/nixos/tests/cardano-node-edge.nix | 5 +- nix/nixos/tests/chairmans-cluster.nix | 11 +- nix/nixos/tests/default.nix | 14 +- nix/pkgs.nix | 91 +-------- nix/scripts.nix | 13 +- nix/sources.json | 62 +------ nix/sources.nix | 246 +++++++++++++------------ nix/stack-shell.nix | 8 +- nix/svclib.nix | 2 +- nix/util.nix | 12 ++ release.nix | 16 +- scripts/buildkite/default.nix | 3 +- scripts/buildkite/stack-cabal-sync.nix | 3 +- scripts/cluster-test.sh | 2 +- shell.nix | 64 ++++++- 24 files changed, 394 insertions(+), 421 deletions(-) delete mode 100644 lib.nix create mode 100644 nix/default.nix create mode 100644 nix/haskell.nix create mode 100644 nix/util.nix diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 68a2540e062..4a16f39fccf 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -12,7 +12,7 @@ steps: system: x86_64-linux - label: 'check-cabal-project' - command: 'nix-build lib.nix -A iohkNix.checkCabalProject -o check-cabal-project.sh && ./check-cabal-project.sh' + command: 'nix-build ./nix -A iohkNix.checkCabalProject -o check-cabal-project.sh && ./check-cabal-project.sh' agents: system: x86_64-linux diff --git a/benchmarking/chain-sync/ci.sh b/benchmarking/chain-sync/ci.sh index 38b14ff6ed5..c9a4e3e1c47 100755 --- a/benchmarking/chain-sync/ci.sh +++ b/benchmarking/chain-sync/ci.sh @@ -19,7 +19,7 @@ LOG_CONFIG="$(yj < $BASEDIR/configuration/log-config-ci.yaml)" CUSTOM_CONFIG="{nodeConfig = builtins.fromJSON ''$LOG_CONFIG'';}" -nix build --out-link ./launch_node -f $BASEDIR/../.. scripts.$CLUSTER.node --arg customConfig "$CUSTOM_CONFIG" +nix build --out-link ./launch_node -f $BASEDIR/../.. scripts.$CLUSTER.node --arg config "{ scriptCustom = $CUSTOM_CONFIG; }" rm -rf "./state-node-$CLUSTER" diff --git a/default.nix b/default.nix index a82d51c1e60..2fcb620d530 100644 --- a/default.nix +++ b/default.nix @@ -1,86 +1,51 @@ { system ? builtins.currentSystem , crossSystem ? null +# allows to cutomize haskellNix (ghc and profiling, see ./nix/haskell.nix) +# and scripts (see ./nix/scripts.nix): , config ? {} +# allows to override dependencies of the project without modifications, +# eg. to test build against local checkout of nixpkgs and iohk-nix: +# nix build -f default.nix cardano-node --arg sourcesOverride '{ +# iohk-nix = ../iohk-nix; +# }' , sourcesOverride ? {} -, profiling ? false -, commonLib ? import ./lib.nix { inherit system crossSystem config profiling; } -, pkgs ? commonLib.pkgs -, customConfig ? {} -, interactive ? false -, gitrev ? commonLib.iohkNix.commitIdFromGitRepoOrZero ./.git -, withHoogle ? true +# pinned version of nixpkgs augmented with overlays (iohk-nix and our packages). +, pkgs ? import ./nix { inherit system crossSystem config sourcesOverride; } +, gitrev ? pkgs.iohkNix.commitIdFromGitRepoOrZero ./.git }: - +with pkgs; with commonLib; let - lib = commonLib.pkgs.lib; - inherit (commonLib) environments haskellPackages niv; - cardano-node = haskellPackages.cardano-node.components.exes.cardano-node; - scripts = commonLib.pkgs.callPackage ./nix/scripts.nix { - inherit commonLib customConfig; - }; + # we are only intersted in listing the project packages + haskellPackages = selectProjectPackages cardanoNodeHaskellPackages; + + scripts = callPackage ./nix/scripts.nix {}; # NixOS tests run a proxy and validate it listens nixosTests = import ./nix/nixos/tests { - inherit (commonLib) pkgs; - inherit commonLib interactive; + inherit pkgs; }; - # we are only intersted in listing the project packages - projectHaskellPackages = commonLib.selectProjectPackages haskellPackages; - - self = with commonLib; { - inherit scripts nixosTests environments cardano-node; + self = { + inherit haskellPackages scripts nixosTests environments check-hydra; - haskellPackages = projectHaskellPackages; - inherit (iohkNix) check-hydra; + inherit (haskellPackages.cardano-node.identifier) version; + # Grab the executable component of our package. + inherit (haskellPackages.cardano-node.components.exes) + cardano-node; # `tests` are the test suites which have been built. - tests = collectComponents' "tests" projectHaskellPackages; + tests = collectComponents' "tests" haskellPackages; # `benchmarks` (only built, not run). - benchmarks = collectComponents' "benchmarks" projectHaskellPackages; - # `checks` collect results of executing the benchmarks and tests: + benchmarks = collectComponents' "benchmarks" haskellPackages; + checks = { - benchmarks = collectChecks self.benchmarks; - tests = collectChecks self.tests; + # `checks.tests` collect results of executing the tests: + tests = collectChecks haskellPackages; } // { recurseForDerivations = true; }; - shell = haskellPackages.shellFor { - - packages = ps: with ps; [ - ps.cardano-node - ps.cardano-config - # in theory we should only have the above two packages (or better, they should be auto-detected), - # but due to source-repository-package declarations being considered as local packages by cabal, we need the following packages as well. - # cf. https://github.com/haskell/cabal/issues/6249 and https://github.com/haskell/cabal/issues/5444 - ps.cardano-sl-x509 - ps.ekg-prometheus-adapter - ps.ouroboros-consensus - ps.ouroboros-network - ]; - - # Builds a Hoogle documentation index of all dependencies, - # and provides a "hoogle" command to search the index. - inherit withHoogle; - - # You might want some extra tools in the shell (optional). - buildInputs = with pkgs; [ - cabal-install - ghcid - hlint - pkgs.haskellPackages.weeder - nix - niv - pkgconfig - sqlite-interactive - tmux - git - ]; - - # Prevents cabal from choosing alternate plans, so that - # *all* dependencies are provided by Nix. - exactDeps = true; + shell = import ./shell.nix { + inherit pkgs; + withHoogle = true; }; - - }; - +}; in self diff --git a/lib.nix b/lib.nix deleted file mode 100644 index 206947761a3..00000000000 --- a/lib.nix +++ /dev/null @@ -1,42 +0,0 @@ -{ system ? builtins.currentSystem -, crossSystem ? null -, config ? {} -, overlays ? [] -, sourcesOverride ? {} -, profiling ? false -}: - -let - # use default stable nixpkgs from iohk-nix instead of our own: - sources = removeAttrs (import ./nix/sources.nix) [ "nixpkgs" ] // sourcesOverride; - iohkNix = import sources.iohk-nix { - inherit config system crossSystem; - sourcesOverride = sources; - nixpkgsOverlays = overlays; - }; - pkgs = iohkNix.pkgs; - haskellPackages = import ./nix/pkgs.nix { - inherit pkgs profiling; - src = ./.; - }; - # TODO: add haskellPackages and fix svclib - svcLib = import ./nix/svclib.nix { inherit pkgs; cardano-node-packages = haskellPackages.cardano-node.components.exes; }; - lib = pkgs.lib; - - collectChecks = lib.mapAttrsRecursiveCond (p: !(lib.isDerivation p)) - (_: p: if (lib.isAttrs p) then pkgs.haskell-nix.haskellLib.check p else p); - - collectComponents' = group: pkgs.haskell-nix.haskellLib.collectComponents group (_:true); - -in lib // iohkNix.cardanoLib // { - inherit (pkgs.haskell-nix.haskellLib) collectComponents selectProjectPackages; - inherit (iohkNix) niv; - inherit - sources - haskellPackages - pkgs - iohkNix - svcLib - collectChecks - collectComponents'; -} diff --git a/nix/default.nix b/nix/default.nix new file mode 100644 index 00000000000..94f754f3daa --- /dev/null +++ b/nix/default.nix @@ -0,0 +1,29 @@ +{ system ? builtins.currentSystem +, crossSystem ? null +, config ? {} +, sourcesOverride ? {} +}: +let + # use default stable nixpkgs from iohk-nix instead of our own: + sources = removeAttrs (import ./sources.nix) [ "nixpkgs" ] + // sourcesOverride; + + # for inclusion in pkgs: + nixpkgsOverlays = [ + (pkgs: _: with pkgs; { + + # mix of pkgs.lib with iohk-nix utils and our own: + commonLib = lib // iohkNix // iohkNix.cardanoLib // + import ./util.nix { inherit haskell-nix; }; + + svcLib = import ./svclib.nix { inherit pkgs; }; + }) + # Our haskell-nix-ified cabal project: + (import ./pkgs.nix) + ]; + + # IOHK pkgs that include haskell-nix overlays, using our sources as override: +in (import sources.iohk-nix { + inherit system crossSystem config nixpkgsOverlays; + sourcesOverride = sources; + }).pkgs diff --git a/nix/haskell.nix b/nix/haskell.nix new file mode 100644 index 00000000000..c6128c51da4 --- /dev/null +++ b/nix/haskell.nix @@ -0,0 +1,77 @@ +############################################################################ +# Builds Haskell packages with Haskell.nix +############################################################################ +{ lib +, stdenv +, haskell-nix +, buildPackages +, config ? {} +# GHC attribute name +, compiler ? config.haskellNix.compiler or "ghc865" +# Enable profiling +, profiling ? config.haskellNix.profiling or false +}: +let + # GHC attribute name + compiler = config.cardanoNode.compiler or "ghc865"; + # Enable profiling + profiling = config.cardanoNode.profiling or false; + + # This creates the Haskell package set. + # https://input-output-hk.github.io/haskell.nix/user-guide/projects/ + pkgSet = haskell-nix.cabalProject { + src = haskell-nix.haskellLib.cleanGit { src = ../.; }; + ghc = buildPackages.haskell-nix.compiler.${compiler}; + modules = [ + + # Allow reinstallation of Win32 + { nonReinstallablePkgs = + [ "rts" "ghc-heap" "ghc-prim" "integer-gmp" "integer-simple" "base" + "deepseq" "array" "ghc-boot-th" "pretty" "template-haskell" + # ghcjs custom packages + "ghcjs-prim" "ghcjs-th" + "ghc-boot" + "ghc" "array" "binary" "bytestring" "containers" + "filepath" "ghc-boot" "ghc-compact" "ghc-prim" + # "ghci" "haskeline" + "hpc" + "mtl" "parsec" "text" "transformers" + "xhtml" + # "stm" "terminfo" + ]; + } + { + # Packages we wish to ignore version bounds of. + # This is similar to jailbreakCabal, however it + # does not require any messing with cabal files. + packages.katip.doExactConfig = true; + + # split data output for ekg to reduce closure size + packages.ekg.components.library.enableSeparateDataOutput = true; + packages.cardano-node.configureFlags = [ "--ghc-option=-Werror" ]; + packages.cardano-config.configureFlags = [ "--ghc-option=-Werror" ]; + enableLibraryProfiling = profiling; + } + (lib.optionalAttrs stdenv.hostPlatform.isWindows { + # Disable cabal-doctest tests by turning off custom setups + packages.comonad.package.buildType = lib.mkForce "Simple"; + packages.distributive.package.buildType = lib.mkForce "Simple"; + packages.lens.package.buildType = lib.mkForce "Simple"; + packages.nonempty-vector.package.buildType = lib.mkForce "Simple"; + packages.semigroupoids.package.buildType = lib.mkForce "Simple"; + + # Make sure we use a buildPackages version of happy + packages.pretty-show.components.library.build-tools = [ buildPackages.haskell-nix.haskellPackages.happy ]; + + # Remove hsc2hs build-tool dependencies (suitable version will be available as part of the ghc derivation) + packages.Win32.components.library.build-tools = lib.mkForce []; + packages.terminal-size.components.library.build-tools = lib.mkForce []; + packages.network.components.library.build-tools = lib.mkForce []; + }) + ]; + # TODO add flags to packages (like cs-ledger) so we can turn off tests that will + # not build for windows on a per package bases (rather than using --disable-tests). + configureArgs = lib.optionalString stdenv.hostPlatform.isWindows "--disable-tests"; + }; +in + pkgSet diff --git a/nix/nixos/cardano-cluster-service.nix b/nix/nixos/cardano-cluster-service.nix index 82dc01dde3b..7df2162294f 100644 --- a/nix/nixos/cardano-cluster-service.nix +++ b/nix/nixos/cardano-cluster-service.nix @@ -15,7 +15,7 @@ in let ### Packages and Nix libs cardano-node = ncfg.package; cardano-sl-pkgs = import ccfg.cardano-sl-src { gitrev = ccfg.cardano-sl-src.rev; }; - svcLib = (import ../svclib.nix { inherit pkgs; }); + svcLib = pkgs.svcLib; in let ### Node enumeration diff --git a/nix/nixos/cardano-node-service.nix b/nix/nixos/cardano-node-service.nix index b455cf9b8a3..17ec36e0b54 100644 --- a/nix/nixos/cardano-node-service.nix +++ b/nix/nixos/cardano-node-service.nix @@ -5,9 +5,9 @@ with lib; with builtins; let - commonLib = import ../../lib.nix {}; + localPkgs = import ../. {}; cfg = config.services.cardano-node; - inherit (commonLib) svcLib; + inherit (localPkgs) svcLib commonLib cardanoNodeHaskellPackages; envConfig = cfg.environments.${cfg.environment}; systemdServiceName = "cardano-node${optionalString cfg.instanced "@"}"; mkScript = cfg: let exec = "cardano-node run"; @@ -59,7 +59,7 @@ in { package = mkOption { type = types.package; - default = commonLib.haskellPackages.cardano-node.components.exes.cardano-node; + default = cardanoNodeHaskellPackages.cardano-node.components.exes.cardano-node; defaultText = "cardano-node"; description = '' The cardano-node package that should be used diff --git a/nix/nixos/chairman-as-a-service.nix b/nix/nixos/chairman-as-a-service.nix index 56f05bc73c5..18b32bb29a6 100644 --- a/nix/nixos/chairman-as-a-service.nix +++ b/nix/nixos/chairman-as-a-service.nix @@ -3,12 +3,12 @@ , pkgs , ... }: -with import ../../lib.nix {}; with lib; with builtins; +with pkgs.commonLib; with lib; with builtins; let + inherit (pkgs) svcLib; cfg = config.services.chairman; ncfg = config.services.cardano-node; - chairman = haskellPackages.cardano-node.components.exes.chairman; - svcLib = (import ../svclib.nix { inherit pkgs; }); + chairman = pkgs.cardanoNodeHaskellPackages.cardano-node.components.exes.chairman; envConfig = environments.${cfg.environment}; mkChairmanConfig = nodeConfig: chairmanConfig: { inherit (nodeConfig) package genesisFile genesisHash stateDir pbftThreshold consensusProtocol; diff --git a/nix/nixos/tests/cardano-node-edge.nix b/nix/nixos/tests/cardano-node-edge.nix index 05a6160ad3f..c834d2947db 100644 --- a/nix/nixos/tests/cardano-node-edge.nix +++ b/nix/nixos/tests/cardano-node-edge.nix @@ -1,9 +1,10 @@ -{ pkgs, commonLib, ... }: - +{ pkgs, ... }: +with pkgs; { name = "cardano-node-edge-test"; nodes = { machine = { config, pkgs, ... }: { + nixpkgs.overlays = pkgsOverlays; imports = [ ../. ]; diff --git a/nix/nixos/tests/chairmans-cluster.nix b/nix/nixos/tests/chairmans-cluster.nix index b0913857c28..d4b8821a1d3 100644 --- a/nix/nixos/tests/chairmans-cluster.nix +++ b/nix/nixos/tests/chairmans-cluster.nix @@ -1,13 +1,13 @@ { pkgs -, commonLib -, interactive ? false +, config +, interactive ? config.interactive or false , ... }: with pkgs.lib; let - svcLib = import ../../svclib.nix { pkgs = commonLib.pkgs; }; - byron-proxy-src = (import ../../sources.nix).cardano-byron-proxy; - cardano-sl-src = (import ../../sources.nix).cardano-sl; + inherit (pkgs) svcLib pkgsOverlays sources; + byron-proxy-src = sources.cardano-byron-proxy; + cardano-sl-src = sources.cardano-sl; # byron-proxy-src = ../../../../cardano-byron-proxy; # cardano-sl-src = ../../../../cardano-sl; cardano-sl-config = pkgs.runCommand "cardano-sl-config" {} '' @@ -35,6 +35,7 @@ in { name = "chairmans-cluster-test"; nodes = { machine = { lib, config, pkgs, ... }: { + nixpkgs.overlays = pkgsOverlays; imports = [ (byron-proxy-src + "/nix/nixos") ../cardano-node-service.nix diff --git a/nix/nixos/tests/default.nix b/nix/nixos/tests/default.nix index 12a70e339cb..ab9a6e68aee 100644 --- a/nix/nixos/tests/default.nix +++ b/nix/nixos/tests/default.nix @@ -1,11 +1,9 @@ -{ pkgs ? (import {}) -, commonLib +{ pkgs , supportedSystems ? [ "x86_64-linux" ] -, interactive ? false }: with pkgs; -with pkgs.lib; +with pkgs.commonLib; let forAllSystems = genAttrs supportedSystems; @@ -13,15 +11,13 @@ with pkgs.lib; imported = import fn; test = import (pkgs.path + "/nixos/tests/make-test.nix") imported; in test ({ - inherit system; + inherit pkgs system config; } // args); callTest = fn: args: forAllSystems (system: hydraJob (importTest fn args system)); in rec { # only tests that port is open since the test can't access network to actually sync - # cardanoNodeEdge = callTest ./cardano-node-edge.nix { inherit commonLib; }; + # cardanoNodeEdge = callTest ./cardano-node-edge.nix {}; # Subsumes what cardanoNodeEdge does - chairmansCluster = callTest ./chairmans-cluster.nix { - inherit commonLib interactive; - }; + chairmansCluster = callTest ./chairmans-cluster.nix {}; } diff --git a/nix/pkgs.nix b/nix/pkgs.nix index 5c32d1d1b34..72b503be6e8 100644 --- a/nix/pkgs.nix +++ b/nix/pkgs.nix @@ -1,82 +1,11 @@ -{ pkgs -, src -, haskellCompiler ? "ghc865" -, profiling ? false -}: -let - - haskell = pkgs.haskell-nix; - - # TODO: move to iohk-nix - # Chop out a subdirectory of the source, so that the package is only - # rebuilt when something in the subdirectory changes. - filterSubDir = dir: with pkgs.lib; let - isFiltered = src ? _isLibCleanSourceWith; - origSrc = if isFiltered then src.origSrc else src; - in cleanSourceWith { - inherit src; - filter = path: type: - type == "directory" || - hasPrefix (toString origSrc + toString dir) path; - } + dir; - - pkg-set = pkgs.haskell-nix.cabalProject { - src = pkgs.haskell-nix.haskellLib.cleanGit { inherit src; }; - ghc = pkgs.buildPackages.haskell-nix.compiler.${haskellCompiler}; - modules = [ - - # Allow reinstallation of Win32 - { nonReinstallablePkgs = - [ "rts" "ghc-heap" "ghc-prim" "integer-gmp" "integer-simple" "base" - "deepseq" "array" "ghc-boot-th" "pretty" "template-haskell" - # ghcjs custom packages - "ghcjs-prim" "ghcjs-th" - "ghc-boot" - "ghc" "array" "binary" "bytestring" "containers" - "filepath" "ghc-boot" "ghc-compact" "ghc-prim" - # "ghci" "haskeline" - "hpc" - "mtl" "parsec" "text" "transformers" - "xhtml" - # "stm" "terminfo" - ]; - } - { - # Packages we wish to ignore version bounds of. - # This is similar to jailbreakCabal, however it - # does not require any messing with cabal files. - packages.katip.doExactConfig = true; - - # split data output for ekg to reduce closure size - packages.ekg.components.library.enableSeparateDataOutput = true; - packages.cardano-node.configureFlags = [ "--ghc-option=-Werror" ]; - packages.cardano-config.configureFlags = [ "--ghc-option=-Werror" ]; - enableLibraryProfiling = profiling; - - # some packages are missing identifier.name: - packages.Win32.package.identifier.name = "Win32"; - packages.file-embed-lzma.package.identifier.name = "file-embed-lzma"; - packages.singletons.package.identifier.name = "singletons"; - } - (pkgs.lib.optionalAttrs pkgs.stdenv.hostPlatform.isWindows { - # Disable cabal-doctest tests by turning off custom setups - packages.comonad.package.buildType = pkgs.lib.mkForce "Simple"; - packages.distributive.package.buildType = pkgs.lib.mkForce "Simple"; - packages.lens.package.buildType = pkgs.lib.mkForce "Simple"; - packages.nonempty-vector.package.buildType = pkgs.lib.mkForce "Simple"; - packages.semigroupoids.package.buildType = pkgs.lib.mkForce "Simple"; - - # Make sure we use a buildPackages version of happy - packages.pretty-show.components.library.build-tools = [ pkgs.buildPackages.haskell-nix.haskellPackages.happy ]; - - # Remove hsc2hs build-tool dependencies (suitable version will be available as part of the ghc derivation) - packages.Win32.components.library.build-tools = pkgs.lib.mkForce []; - packages.terminal-size.components.library.build-tools = pkgs.lib.mkForce []; - packages.network.components.library.build-tools = pkgs.lib.mkForce []; - }) - ]; - # TODO add flags to packages (like cs-ledger) so we can turn off tests that will - # not build for windows on a per package bases (rather than using --disable-tests). - configureArgs = pkgs.lib.optionalString pkgs.stdenv.hostPlatform.isWindows "--disable-tests"; +# our packages overlay +pkgs: _: with pkgs; { + cardanoNodeHaskellPackages = import ./haskell.nix { + inherit config + lib + stdenv + haskell-nix + buildPackages + ; }; -in pkg-set +} diff --git a/nix/scripts.nix b/nix/scripts.nix index 0c5f0cf6e9b..ea5a22e5037 100644 --- a/nix/scripts.nix +++ b/nix/scripts.nix @@ -1,7 +1,10 @@ -{ commonLib, customConfig }: -with commonLib.pkgs.lib; +{ pkgs +, config +, customConfig ? config.scriptCustom or {} + }: +with pkgs.commonLib; let - inherit (commonLib) svcLib pkgs; + inherit (pkgs) svcLib; pkgsModule = { config._module.args.pkgs = mkDefault pkgs; }; @@ -41,7 +44,7 @@ let edgePort = if config.useProxy then config.proxyPort else config.edgePort; edgeHost = if config.useProxy then config.proxyHost else config.edgeHost; edgeNodes = if config.useProxy then [] else config.edgeNodes; - in config.topologyFile or commonLib.mkEdgeTopology { + in config.topologyFile or mkEdgeTopology { inherit (config) hostAddr port; inherit edgeNodes edgeHost edgePort; }; @@ -80,7 +83,7 @@ let cd "state-node-${envConfig.name}" ${nodeScript} $@ ''; - scripts = commonLib.forEnvironments (environment: + scripts = forEnvironments (environment: { node = mkNodeScript environment; chairman = svcLib.mkChairmanScript; diff --git a/nix/sources.json b/nix/sources.json index 67b5b64f6fd..551db19fb29 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -23,64 +23,16 @@ "url": "https://github.com/input-output-hk/cardano-sl/archive/d0cc2e6336e3c57771c775cafe831bc83db303d3.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, - "gitignore": { - "branch": "master", - "description": "Nix function for filtering local git sources", - "homepage": "", - "owner": "hercules-ci", - "repo": "gitignore", - "rev": "f9e996052b5af4032fe6150bba4a6fe4f7b9d698", - "sha256": "0jrh5ghisaqdd0vldbywags20m2cxpkbbk5jjjmwaw0gr8nhsafv", - "type": "tarball", - "url": "https://github.com/hercules-ci/gitignore/archive/f9e996052b5af4032fe6150bba4a6fe4f7b9d698.tar.gz", - "url_template": "https://github.com///archive/.tar.gz" - }, - "hackage.nix": { - "branch": "master", - "description": "Automatically generated Nix expressions for Hackage", - "homepage": "", - "owner": "input-output-hk", - "repo": "hackage.nix", - "rev": "3c95c2c481ca909ecdb85354348482edf924df3a", - "sha256": "0sk5810bs6w51vq2501bbbx2adcf4ygkw0x3w014l7kpds6fvxpy", - "type": "tarball", - "url": "https://github.com/input-output-hk/hackage.nix/archive/3c95c2c481ca909ecdb85354348482edf924df3a.tar.gz", - "url_template": "https://github.com///archive/.tar.gz" - }, - "haskell.nix": { - "branch": "master", - "description": "Alternative Haskell Infrastructure for Nixpkgs", - "homepage": "https://input-output-hk.github.io/haskell.nix", - "owner": "input-output-hk", - "repo": "haskell.nix", - "rev": "fb49ef450107d23134b45b81f96a34b3c0a8c8cd", - "sha256": "1kxdwkc4lxfkfz22b4ghmf9nhafpyja167dmdq75wvj1xjadx2hd", - "type": "tarball", - "url": "https://github.com/input-output-hk/haskell.nix/archive/fb49ef450107d23134b45b81f96a34b3c0a8c8cd.tar.gz", - "url_template": "https://github.com///archive/.tar.gz" - }, "iohk-nix": { - "branch": "master", + "branch": "update-nixpkgs-haskell-nix", "description": "nix scripts shared across projects", "homepage": null, "owner": "input-output-hk", "repo": "iohk-nix", - "rev": "493582154fbbe03563056a263bff729c3a8cee1c", - "sha256": "064pa40w1j72kxbp99ca31hi88hdpy10vk8i6g35ii10242qw2b5", - "type": "tarball", - "url": "https://github.com/input-output-hk/iohk-nix/archive/493582154fbbe03563056a263bff729c3a8cee1c.tar.gz", - "url_template": "https://github.com///archive/.tar.gz" - }, - "niv": { - "branch": "iohk", - "description": "Easy dependency management for Nix projects", - "homepage": "https://github.com/nmattia/niv", - "owner": "input-output-hk", - "repo": "niv", - "rev": "c25268460b72cea5b7294354ab019997530a6f3d", - "sha256": "1fgq1nkycm54f431h6wflh6rr7xsq3q2fq9drbgra2fnrsyn65ll", + "rev": "ff4c98edc2885b97b8547ba6d3507d29fed9c935", + "sha256": "0ih5fp9rj5ps2x2n41nhfbq2h3p6vj8jglva448j71ywqkz5vjr6", "type": "tarball", - "url": "https://github.com/input-output-hk/niv/archive/c25268460b72cea5b7294354ab019997530a6f3d.tar.gz", + "url": "https://github.com/input-output-hk/iohk-nix/archive/ff4c98edc2885b97b8547ba6d3507d29fed9c935.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "nixpkgs": { @@ -89,10 +41,10 @@ "homepage": null, "owner": "NixOS", "repo": "nixpkgs", - "rev": "f618cfbf4544b09088cbbb6004cfc84dd432e20c", - "sha256": "1272psriz048bc1ab4vjkd0qq942kifalraq0j9km62cgza1mf5h", + "rev": "ea553d8c67c6a718448da50826ff5b6916dc9c59", + "sha256": "0g9smv36sk42rfyzi8wyq2wl11c5l0qaldij1zjdj60s57cl3wgj", "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/f618cfbf4544b09088cbbb6004cfc84dd432e20c.tar.gz", + "url": "https://github.com/NixOS/nixpkgs/archive/ea553d8c67c6a718448da50826ff5b6916dc9c59.tar.gz", "url_template": "https://github.com///archive/.tar.gz" } } diff --git a/nix/sources.nix b/nix/sources.nix index b7be13a8a7e..8a725cb4e7f 100644 --- a/nix/sources.nix +++ b/nix/sources.nix @@ -1,130 +1,134 @@ # This file has been generated by Niv. -# A record, from name to path, of the third-party packages -with rec -{ - pkgs = - if hasNixpkgsPath - then - if hasThisAsNixpkgsPath - then import (builtins_fetchTarball { inherit (sources_nixpkgs) url sha256; }) {} - else import {} +let + + # + # The fetchers. fetch_ fetches specs of type . + # + + fetch_file = pkgs: spec: + if spec.builtin or true then + builtins_fetchurl { inherit (spec) url sha256; } + else + pkgs.fetchurl { inherit (spec) url sha256; }; + + fetch_tarball = pkgs: spec: + if spec.builtin or true then + builtins_fetchTarball { inherit (spec) url sha256; } + else + pkgs.fetchzip { inherit (spec) url sha256; }; + + fetch_git = spec: + builtins.fetchGit { url = spec.repo; inherit (spec) rev ref; }; + + fetch_builtin-tarball = spec: + builtins.trace + '' + WARNING: + The niv type "builtin-tarball" will soon be deprecated. You should + instead use `builtin = true`. + + $ niv modify -a type=tarball -a builtin=true + '' + builtins_fetchTarball { inherit (spec) url sha256; }; + + fetch_builtin-url = spec: + builtins.trace + '' + WARNING: + The niv type "builtin-url" will soon be deprecated. You should + instead use `builtin = true`. + + $ niv modify -a type=file -a builtin=true + '' + (builtins_fetchurl { inherit (spec) url sha256; }); + + # + # Various helpers + # + + # The set of packages used when specs are fetched using non-builtins. + mkPkgs = sources: + let + sourcesNixpkgs = + import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) {}; + hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath; + hasThisAsNixpkgsPath = == ./.; + in + if builtins.hasAttr "nixpkgs" sources + then sourcesNixpkgs + else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then + import {} + else + abort + '' + Please specify either (through -I or NIX_PATH=nixpkgs=...) or + add a package called "nixpkgs" to your sources.json. + ''; + + # The actual fetching function. + fetch = pkgs: name: spec: + + if ! builtins.hasAttr "type" spec then + abort "ERROR: niv spec ${name} does not have a 'type' attribute" + else if spec.type == "file" then fetch_file pkgs spec + else if spec.type == "tarball" then fetch_tarball pkgs spec + else if spec.type == "git" then fetch_git spec + else if spec.type == "builtin-tarball" then fetch_builtin-tarball spec + else if spec.type == "builtin-url" then fetch_builtin-url spec else - import (builtins_fetchTarball { inherit (sources_nixpkgs) url sha256; }) {}; - - sources_nixpkgs = - if builtins.hasAttr "nixpkgs" sources - then sources.nixpkgs - else abort - '' - Please specify either (through -I or NIX_PATH=nixpkgs=...) or - add a package called "nixpkgs" to your sources.json. - ''; - - sources_gitignore = - if builtins.hasAttr "gitignore" sources - then sources.gitignore - else abort - '' - Please add "gitignore" to your sources.json: - niv add hercules-ci/gitignore - ''; - - inherit (import (builtins_fetchTarball { inherit (sources_gitignore) url sha256; }) { - inherit (pkgs) lib; - }) gitignoreSource; + abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}"; + + # Ports of functions for older nix versions + + # a Nix version of mapAttrs if the built-in doesn't exist + mapAttrs = builtins.mapAttrs or ( + f: set: with builtins; + listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set)) + ); # fetchTarball version that is compatible between all the versions of Nix - builtins_fetchTarball = - { url, sha256 ? null }@attrs: - let - inherit (builtins) lessThan nixVersion fetchTarball; - in - if sha256 == null || lessThan nixVersion "1.12" then - fetchTarball { inherit url; } - else - fetchTarball attrs; + builtins_fetchTarball = { url, sha256 }@attrs: + let + inherit (builtins) lessThan nixVersion fetchTarball; + in + if lessThan nixVersion "1.12" then + fetchTarball { inherit url; } + else + fetchTarball attrs; # fetchurl version that is compatible between all the versions of Nix - builtins_fetchurl = - { url, sha256 ? null }@attrs: - let - inherit (builtins) lessThan nixVersion fetchurl; - in - if sha256 == null || lessThan nixVersion "1.12" then - fetchurl { inherit url; } + builtins_fetchurl = { url, sha256 }@attrs: + let + inherit (builtins) lessThan nixVersion fetchurl; + in + if lessThan nixVersion "1.12" then + fetchurl { inherit url; } + else + fetchurl attrs; + + # Create the final "sources" from the config + mkSources = config: + mapAttrs ( + name: spec: + if builtins.hasAttr "outPath" spec + then abort + "The values in sources.json should not have an 'outPath' attribute" else - fetchurl attrs; - - # A wrapper around pkgs.fetchzip that has inspectable arguments, - # annoyingly this means we have to specify them - fetchzip = { url, sha256 ? null }@attrs: if sha256 == null - then builtins.fetchTarball { inherit url; } - else pkgs.fetchzip attrs; - - # A wrapper around pkgs.fetchurl that has inspectable arguments, - # annoyingly this means we have to specify them - fetchurl = { url, sha256 }@attrs: pkgs.fetchurl attrs; - - hasNixpkgsPath = (builtins.tryEval ).success; - hasThisAsNixpkgsPath = - (builtins.tryEval ).success && == ./.; - - sources = builtins.fromJSON (builtins.readFile ./sources.json); - - mapAttrs = builtins.mapAttrs or - (f: set: with builtins; - listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set))); - - # borrowed from nixpkgs - functionArgs = f: f.__functionArgs or (builtins.functionArgs f); - callFunctionWith = autoArgs: f: args: - let auto = builtins.intersectAttrs (functionArgs f) autoArgs; - in f (auto // args); - - getFetcher = spec: - let fetcherName = - if builtins.hasAttr "type" spec - then builtins.getAttr "type" spec - else "builtin-tarball"; - in builtins.getAttr fetcherName { - "tarball" = fetchzip; - "builtin-tarball" = builtins_fetchTarball; - "file" = fetchurl; - "builtin-url" = builtins_fetchurl; + spec // { outPath = fetch config.pkgs name spec; } + ) config.sources; + + # The "config" used by the fetchers + mkConfig = + { sourcesFile ? ./sources.json + , sources ? builtins.fromJSON (builtins.readFile sourcesFile) + , pkgs ? mkPkgs sources + }: rec { + # The sources, i.e. the attribute set of spec name to spec + inherit sources; + + # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers + inherit pkgs; }; -}; -# NOTE: spec must _not_ have an "outPath" attribute -mapAttrs (name: spec: - if builtins.hasAttr "outPath" spec - then abort - "The values in sources.json should not have an 'outPath' attribute" - else - let - host = if (name == "nixpkgs") then "custom_nixpkgs" else name; - tryFromPath = builtins.tryEval (builtins.findFile builtins.nixPath host); - defaultSpec = (if builtins.hasAttr "url" spec && builtins.hasAttr "sha256" spec - then spec // - { outPath = callFunctionWith spec (getFetcher spec) { }; } - else spec) // (if tryFromPath.success - then let path = tryFromPath.value; - in { - outPath = builtins.trace "using search host <${host}>" ( - if pkgs.lib.hasPrefix "/nix/store" (builtins.toString path) - then path else gitignoreSource path); - } - else {}); - in if builtins.hasAttr "rev" spec && builtins.hasAttr "url" spec then - defaultSpec // - { revOverride = rev: if (rev == null) then defaultSpec else - let - spec' = removeAttrs (spec // { - rev = rev; - url = builtins.replaceStrings [defaultSpec.rev] [rev] defaultSpec.url; - }) [ "sha256" ]; - in - spec' // - { outPath = callFunctionWith spec' (getFetcher spec') { }; }; - } - else defaultSpec - ) sources +in +mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); } diff --git a/nix/stack-shell.nix b/nix/stack-shell.nix index 4b66d2c6907..4199ab0b736 100644 --- a/nix/stack-shell.nix +++ b/nix/stack-shell.nix @@ -1,8 +1,8 @@ -with import ../lib.nix {}; +with import ./. {}; -pkgs.haskell.lib.buildStackProject { - name = "cardano-node-stack-env"; +haskell.lib.buildStackProject { + name = "stack-env"; buildInputs = with pkgs; [ zlib openssl gmp libffi git systemd haskellPackages.happy ]; - ghc = (import ../shell.nix {}).ghc; + ghc = (import ../shell.nix {inherit pkgs;}).ghc; } diff --git a/nix/svclib.nix b/nix/svclib.nix index 642169320e2..1be526e0668 100644 --- a/nix/svclib.nix +++ b/nix/svclib.nix @@ -1,4 +1,4 @@ -{ pkgs, cardano-node-packages ? (import ../lib.nix {}).haskellPackages.cardano-node.components.exes }: +{ pkgs, cardano-node-packages ? pkgs.cardanoNodeHaskellPackages.cardano-node.components.exes }: with builtins; with pkgs.lib; let cardano-cli-bin = "${cardano-node-packages.cardano-cli}/bin/cardano-cli"; diff --git a/nix/util.nix b/nix/util.nix new file mode 100644 index 00000000000..06f6f72484c --- /dev/null +++ b/nix/util.nix @@ -0,0 +1,12 @@ +{ haskell-nix }: + +with haskell-nix.haskellLib; +{ + + inherit selectProjectPackages; + + inherit (extra) + collectComponents' + collectChecks; + +} diff --git a/release.nix b/release.nix index 65862d27d14..1969f7d9f7a 100644 --- a/release.nix +++ b/release.nix @@ -25,13 +25,16 @@ # A Hydra option , scrubJobs ? true -# Import IOHK common nix lib -, commonLib ? import ./lib.nix {} +# Dependencies overrides +, sourcesOverride ? {} + +# Import pkgs, including IOHK common nix lib +, pkgs ? import ./nix { inherit sourcesOverride; } }: -with (import commonLib.iohkNix.release-lib) { - inherit (commonLib) pkgs; +with (import pkgs.iohkNix.release-lib) { + inherit pkgs; inherit supportedSystems supportedCrossSystems scrubJobs projectArgs; packageSet = import cardano-node; gitrev = cardano-node.rev; @@ -60,7 +63,7 @@ let # TODO: add docker images #wrapDockerImage = cluster: let # images = (getArchDefault "x86_64-linux").dockerImages; - # wrapImage = image: commonLib.pkgs.runCommand "${image.name}-hydra" {} '' + # wrapImage = image: pkgs.runCommand "${image.name}-hydra" {} '' # mkdir -pv $out/nix-support/ # cat < $out/nix-support/hydra-build-products # file dockerimage ${image} @@ -104,7 +107,7 @@ let # TODO: fix broken evals #musl64 = mapTestOnCross musl64 (packagePlatformsCross project); } // extraBuilds // (mkRequiredJob ( - collectTests jobs.native.checks.tests ++ + collectTests jobs.native.checks ++ collectTests jobs.native.benchmarks ++ [ jobs.native.cardano-node.x86_64-darwin jobs.native.cardano-node.x86_64-linux @@ -112,7 +115,6 @@ let # windows cross compilation targets #jobs.x86_64-pc-mingw32.cardano-node.x86_64-linux - # Chairman consensus test TODO: fix the chairman test jobs.nixosTests.chairmansCluster.x86_64-linux ])); diff --git a/scripts/buildkite/default.nix b/scripts/buildkite/default.nix index 26f5562179b..806ee3db40e 100644 --- a/scripts/buildkite/default.nix +++ b/scripts/buildkite/default.nix @@ -1,5 +1,4 @@ -with import ../../lib.nix {}; -with pkgs; +with import ../../nix {}; let stack-hpc-coveralls = iohkNix.stack-hpc-coveralls; diff --git a/scripts/buildkite/stack-cabal-sync.nix b/scripts/buildkite/stack-cabal-sync.nix index 652bb775f96..a56699f38cf 100644 --- a/scripts/buildkite/stack-cabal-sync.nix +++ b/scripts/buildkite/stack-cabal-sync.nix @@ -1,8 +1,7 @@ { system ? builtins.currentSystem }: let - self = import ../../lib.nix { inherit system; }; - pkgs = self.pkgs; + pkgs = import ../../nix { inherit system; }; repo-tool-src = pkgs.fetchFromGitHub { owner = "input-output-hk"; repo = "cardano-repo-tool"; diff --git a/scripts/cluster-test.sh b/scripts/cluster-test.sh index 1c3d443f796..e9bfa7f4e6a 100755 --- a/scripts/cluster-test.sh +++ b/scripts/cluster-test.sh @@ -59,5 +59,5 @@ EOF announce trap announce EXIT -nix-build -A nixosTests.chairmansCluster --show-trace --arg interactive true "$@" 2>&1 | +nix-build -A nixosTests.chairmansCluster --show-trace --arg config '{ interactive = true; }' "$@" 2>&1 | tee ${logfile} diff --git a/shell.nix b/shell.nix index 465cf24a9a5..2501893f683 100644 --- a/shell.nix +++ b/shell.nix @@ -1,19 +1,63 @@ -{ commonLib ? import ./lib.nix {} +# This file is used by nix-shell. +# It just takes the shell attribute from default.nix. +{ config ? {} +, sourcesOverride ? {} , withHoogle ? false +, pkgs ? import ./nix { + inherit config sourcesOverride; + } }: +with pkgs; let - pkgs = commonLib.pkgs; - default = import ./. { inherit withHoogle; }; - shell = default.shell; + # This provides a development environment that can be used with nix-shell or + # lorri. See https://input-output-hk.github.io/haskell.nix/user-guide/development/ + shell = cardanoNodeHaskellPackages.shellFor { + name = "cabal-dev-shell"; + + # If shellFor local packages selection is wrong, + # then list all local packages then include source-repository-package that cabal complains about: + packages = ps: with ps; [ + ps.cardano-node + ps.cardano-config + # in theory we should only have the above two packages (or better, they should be auto-detected), + # but due to source-repository-package declarations being considered as local packages by cabal, we need the following packages as well. + # cf. https://github.com/haskell/cabal/issues/6249 and https://github.com/haskell/cabal/issues/5444 + ps.cardano-sl-x509 + ps.ekg-prometheus-adapter + ps.ouroboros-consensus + ps.ouroboros-network + ]; + + # These programs will be available inside the nix-shell. + buildInputs = with haskellPackages; [ + cabal-install + ghcid + hlint + weeder + nix + niv + pkgconfig + sqlite-interactive + tmux + git + ]; + + # Prevents cabal from choosing alternate plans, so that + # *all* dependencies are provided by Nix. + exactDeps = true; + + inherit withHoogle; + }; + devops = pkgs.stdenv.mkDerivation { name = "devops-shell"; buildInputs = [ - commonLib.niv + niv ]; shellHook = '' echo "DevOps Tools" \ - | ${pkgs.figlet}/bin/figlet -f banner -c \ - | ${pkgs.lolcat}/bin/lolcat + | ${figlet}/bin/figlet -f banner -c \ + | ${lolcat}/bin/lolcat echo "NOTE: you may need to export GITHUB_TOKEN if you hit rate limits with niv" echo "Commands: @@ -22,5 +66,7 @@ let " ''; }; -# TODO: switch back to default.nix shell when it works -in default.shell // { inherit devops; } + +in + + shell // { inherit devops; }