Various improvements to the dynamically generated Nix versions

- Use separate JSON files to track sources for Nixpkgs and Nix releases This greatly simplifies the code, because we can directly encode versions as keys without ambiguity - Avoid an instance of IFD for the redirect generation - Use pkgs.substitute instead of builtins.replaceStrings - Turn off the single-page feature for now. It was added by Valentin in a previous commit, but I think we should discuss this a bit more - Simplify a lot of the code and add comments - Fix the mutable redirects, they were broken by a parent commit - Remove pieces of Nix code that weren't used, like the import of the Nixpkgs manual. This can be added in the future when necessary - Make sure that Nix versions are cached by building from the store path
fricklerhandwerk · Apr 4, 2024 · f479463 · f479463
1 parent e837576
commit f479463
Show file tree

Hide file tree

Showing 9 changed files with 453 additions and 547 deletions.
diff --git a/default.nix b/default.nix
@@ -1,9 +1,9 @@
-{ inputs ? import ./nix/sources.nix
+{ inputs ? import ./nix/inputs.nix
 , system ? builtins.currentSystem
 ,
 }:
 let
-  pkgs = import inputs.nixpkgs_23-05 {
+  pkgs = import inputs.nixpkgs."23.05" {
     config = { };
     overlays = [ (import ./nix/overlay.nix) ];
     inherit system;
@@ -13,10 +13,6 @@ let
   releases = import ./nix/releases.nix { inherit lib inputs system; };
 
   nix-dev =
-    let
-      # Various versions of the Nix manuals, grep for (nix-manual)= to find where they are displayed.
-      # FIXME: This requires human interaction to update! See ./CONTRIBUTING.md for details.
-    in
     pkgs.stdenv.mkDerivation {
       name = "nix-dev";
       src = ./.;
@@ -33,54 +29,42 @@ let
       ];
       buildPhase =
         let
-          nix-manual-index =
-            let
-              inherit (lib.strings) replaceStrings;
-              inherit (lib.attrsets) mapAttrsToList;
-              inherit (releases) version supported-releases;
-            in
-            replaceStrings
-              (mapAttrsToList (release: _: "@${release}@") supported-releases)
-              (mapAttrsToList (_: package: version package) supported-releases)
-              (builtins.readFile ./source/reference/nix-manual.md);
+          substitutedNixManualReference = pkgs.substitute {
+            src = ./source/reference/nix-manual.md;
+            replacements = lib.concatLists (lib.mapAttrsToList (from: to: [ "--subst-var-by" from to ]) releases.substitutions);
+          };
         in
         ''
-          cp -f ${builtins.toFile "nix-manual.md" nix-manual-index} $TMP/nix.dev/source/reference/nix-manual.md
+          cp -f ${substitutedNixManualReference} source/reference/nix-manual.md
           make html
         '';
       installPhase =
         let
-          inherit (releases) version;
-          copy = nix: ''
-            cp -Rf ${nix.doc}/share/doc/nix/manual/* $out/manual/nix/${version nix}
-          '';
-          # provide a single-page view from mdBook's print feature
-          single-page = nix:
-            ''
-              sed -z 's|\s*window\.addEventListener(\x27load\x27, function() {\s*window\.setTimeout(window.print, 100);\s*});||g' ${nix.doc}/share/doc/nix/manual/print.html > $out/manual/nix/${version nix}/nix-${version nix}.html
-            '';
-          # add upstream page redirects of the form `<from> <to> <status>`, excluding comments and empty lines
-          redirects = nix:
+          # Various versions of the Nix manuals, grep for (nix-manual)= to find where they are displayed.
+          # FIXME: This requires human interaction to update! See ./CONTRIBUTING.md for details.
+          release = version: nix: ''
+            cp -R --no-preserve=mode ${nix.doc}/share/doc/nix/manual $out/manual/nix/${version}
+
+            # add upstream page redirects of the form `<from> <to> <status>`, excluding comments and empty lines
             # not all releases have that though
-            lib.optionalString (lib.pathExists "${nix.doc}/share/doc/nix/manual/_redirects") ''
-              sed '/^#/d;/^$/d;s#^\(.*\) \(.*\) #/manual/nix/${version nix}\1 /manual/nix/${version nix}\2 #g' ${nix.doc}/share/doc/nix/manual/_redirects >> $out/_redirects
-            '';
-          shortlink = release: nix: ''
-            echo /nix/manual/${release}/* /nix/manual/${version nix}/:splat 302 >> $out/_redirects
+            if [[ -f ${nix.doc}/share/doc/nix/manual/_redirects ]]; then
+              sed '/^#/d;/^$/d;s#^\(.*\) \(.*\) #/manual/nix/${version}\1 /manual/nix/${version}\2 #g' ${nix.doc}/share/doc/nix/manual/_redirects >> $out/_redirects
+            fi
+          '';
+          # TODO: This needs to be looked into more carefully before enabling:
+          # This is a hacky way to get a single-page manual from mdBook's print feature:
+          # sed -z 's|\s*window\.addEventListener(\x27load\x27, function() {\s*window\.setTimeout(window.print, 100);\s*});||g' ${nix.doc}/share/doc/nix/manual/print.html > $out/manual/nix/${version}/sp.html
+
+          # Redirects from mutable URLs like /manual/nix/latest/... to /manual/nix/2.21/...
+          mutableRedirect = mutable: immutable: ''
+            echo "/manual/nix/${mutable}/* /manual/nix/${immutable}/:splat 302" >> $out/_redirects
           '';
-          inherit (releases) unique-versions nix-releases;
-          inherit (lib.attrsets) mapAttrsToList attrValues;
-          inherit (lib.strings) concatStringsSep;
         in
         ''
-          mkdir -p $out
+          mkdir -p $out/manual/nix
           cp -R build/html/* $out/
-          # NOTE: the comma in the shell expansion makes it also work for singleton lists
-          mkdir -p $out/manual/nix/{${concatStringsSep "," (mapAttrsToList (_: nix: version nix) nix-releases)},}
-          ${concatStringsSep "\n" (map copy (unique-versions (attrValues nix-releases)))}
-          ${concatStringsSep "\n" (map single-page (unique-versions (attrValues nix-releases)))}
-          ${concatStringsSep "\n" (map redirects (unique-versions (attrValues nix-releases)))}
-          ${concatStringsSep "\n" (mapAttrsToList shortlink nix-releases)}
+          ${lib.concatStringsSep "\n" (lib.mapAttrsToList release releases.nixReleases)}
+          ${lib.concatStringsSep "\n" (lib.mapAttrsToList mutableRedirect releases.mutableNixManualRedirects)}
         '';
     };
 
@@ -131,6 +115,7 @@ in
 {
   # build with `nix-build -A build`
   build = nix-dev;
+
   shell = pkgs.mkShell {
     inputsFrom = [ nix-dev ];
     packages = [
@@ -142,7 +127,4 @@ in
       pkgs.vale
     ];
   };
-  nix_2-15 = (import inputs.nix_2-15).default;
-  nix_2-16 = (import inputs.nix_2-16).default;
-  nix_2-14 = (import inputs.nix_2-14).default;
 }
diff --git a/nix/inputs.nix b/nix/inputs.nix
@@ -0,0 +1,36 @@
+# An abstraction over the different source pins in this directory
+{
+
+  # The main niv pins, these are without any standard names:
+  # {
+  #   <name> = <source>;
+  # }
+  main = import ./sources.nix {
+    sourcesFile = ./sources.json;
+  };
+
+  # Sources for Nix releases, the attribute name is the release version.
+  # These are done specially because updating these is non-trivial.
+  # See ./update-nix-releases.nix
+  # {
+  #   <version> = <source>;
+  # }
+  nix =
+    builtins.mapAttrs (name: value:
+      # This matches the nix-prefetch-url --unpack --name source call in ./update-nix-releases.nix
+      fetchTarball {
+        name = "source";
+        url = value.url;
+        sha256 = value.sha256;
+      }
+    ) (builtins.fromJSON (builtins.readFile ./nix-versions.json));
+
+  # Sources for Nixpkgs releases, the attribute name is the release name.
+  # These can be updated with the standard niv tooling, but are tracked separately to avoid having to filter them out during processing.
+  # See ./update-nixpkgs-releases.nix
+  nixpkgs =
+    import ./sources.nix {
+      sourcesFile = ./nixpkgs-versions.json;
+    };
+
+}
diff --git a/nix/nix-versions.json b/nix/nix-versions.json
@@ -0,0 +1,92 @@
+{
+  "2.21": {
+    "url": "https://github.com/nixos/nix/tarball/92e38fe30ccd7ca7e872b5da3b0fc138fed3b438",
+    "version": "2.21.2",
+    "sha256": "1wdaq3iv4dkajnvp4mn6ykq1f3pb24kgls3vggch3h08aadd2nj6"
+  },
+  "2.20": {
+    "url": "https://github.com/nixos/nix/tarball/7bc4af7301df34ea4e157338ac3792c1a9ae35b7",
+    "version": "2.20.6",
+    "sha256": "17mpp5s3045zmilhvysa29g44clcfi5wn41sslkhxm7a50k7qa85"
+  },
+  "2.19": {
+    "url": "https://github.com/nixos/nix/tarball/82f44d8633c849497d64980b5705d09ddbb01164",
+    "version": "2.19.5",
+    "sha256": "03vwahd1v8vq580cksr2grspj7iibdh5adyxdff8gq32scxahqzz"
+  },
+  "2.18": {
+    "url": "https://github.com/nixos/nix/tarball/76364a847dd39e77ad5eebfcfa86df25f518ab1f",
+    "version": "2.18.3",
+    "sha256": "1d7b3khvkrmq3m34vw75l52a83rc7f2c4al7c5nyggwclkmnh11w"
+  },
+  "2.17": {
+    "url": "https://github.com/nixos/nix/tarball/27c8b92f31fc4fb1d5be28d5196dc5ac0da58aee",
+    "version": "2.17.3",
+    "sha256": "1v1nw39j67y34hjnh9126975j1l1ha4ysfr70mnvpvsl3zb6mjvv"
+  },
+  "2.16": {
+    "url": "https://github.com/nixos/nix/tarball/d9c204fafcbb5a05aef5c36fc83f4c3d6cc4749d",
+    "version": "2.16.4",
+    "sha256": "0m3x4qgp2m61d1dd5mfqwxw5r0dgy7pzdc7zg0z9x9l7slghx3ma"
+  },
+  "2.15": {
+    "url": "https://github.com/nixos/nix/tarball/b59275b4417afd459dcf08a097fbd659f63161c2",
+    "version": "2.15.4",
+    "sha256": "0wx0z9b8abhdxcdm10mahx9dqqmwm63yrpmb5ax8ykzl7ij58wgv"
+  },
+  "2.14": {
+    "url": "https://github.com/nixos/nix/tarball/9e19aca31bc9237d699b86bce1d31aa099f09121",
+    "version": "2.14.1",
+    "sha256": "1wnaah9bpycl89bjv85b967vxvgrppfl1a8f1h7vvavhb2fcgxkb"
+  },
+  "2.13": {
+    "url": "https://github.com/nixos/nix/tarball/17e5d0325d0d7bb108d35f678622e3c237ca9fe9",
+    "version": "2.13.7",
+    "sha256": "1brlmfm2ab37hzsi5p6m13gpfhqzqkr1nvwkg2ks8ij1mcz7dh67"
+  },
+  "2.12": {
+    "url": "https://github.com/nixos/nix/tarball/3c1f09ca492f5ec2252b511c9b627eddd2e98f88",
+    "version": "2.12.2",
+    "sha256": "14bjd85i3qm8jzzyafv1jcsdq5ylrp96563141dimc7p2085jczs"
+  },
+  "2.11": {
+    "url": "https://github.com/nixos/nix/tarball/1aac1884c5d5976cc74a0a4e193e0b0ca3f25dbf",
+    "version": "2.11.2",
+    "sha256": "0ivcvz7g838sbh05vgzirz0agbr1dbr3gw1ywmxzmm3bh75hw2gs"
+  },
+  "2.10": {
+    "url": "https://github.com/nixos/nix/tarball/73b3ba878a86e9a4d5e46fb16ce735b7dea4c929",
+    "version": "2.10.3",
+    "sha256": "0s7i6mj3x9l5l3b73fp4jq0cl1sjah7lq11gkymzb2jrqhxc681z"
+  },
+  "2.9": {
+    "url": "https://github.com/nixos/nix/tarball/338418790adac11a4e73218e40fa109c4cfbc86d",
+    "version": "2.9.3",
+    "sha256": "0kl36c365dd9mzjcr5py52lgm78hj81cvw7lws9f16zkj64hcy9v"
+  },
+  "2.8": {
+    "url": "https://github.com/nixos/nix/tarball/4555784afb613247563896273dad7887c1a2a0a3",
+    "version": "2.8.2",
+    "sha256": "0zg7y4c1fjz241fspzmba3acjw1qbigvkngp7q2xb5mdm5sl6swp"
+  },
+  "2.7": {
+    "url": "https://github.com/nixos/nix/tarball/5ed3a9db6add8b28bdb484c00f45033f50ccb853",
+    "version": "2.7.0",
+    "sha256": "03y59yxc32ghkfrfkrrjkhpnksyp5q29mlhq1ssxia29wjmv5mz3"
+  },
+  "2.6": {
+    "url": "https://github.com/nixos/nix/tarball/e044ccb67ce38d11059de32515306f5f1bd2f04f",
+    "version": "2.6.1",
+    "sha256": "0pfx87zgyhhhf4mnrknzq0pgry8i2d5pxvsbf52sqrxxzznr1n0k"
+  },
+  "2.5": {
+    "url": "https://github.com/nixos/nix/tarball/16b3e41ed0e4b6c2ceffcbe8956acdd8b0cde71d",
+    "version": "2.5.1",
+    "sha256": "1px45lab0yb9gx6bysr296w3bc6cv1cm3x3lik1hhi3k5fpfc3pn"
+  },
+  "2.4": {
+    "url": "https://github.com/nixos/nix/tarball/fbe3b09248feb251f7baf24dc5cd6d675ec0cb2c",
+    "version": "2.4.1",
+    "sha256": "0kzn3ggmir8iv1nk4yds6676x8y4q72xv06ks5p2nrvazhdarl8y"
+  }
+}
diff --git a/nix/nixpkgs-versions.json b/nix/nixpkgs-versions.json
@@ -0,0 +1,134 @@
+{
+    "18.09": {
+        "branch": "nixos-18.09",
+        "description": "Nix Packages collection & NixOS",
+        "homepage": "",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "a7e559a5504572008567383c3dc8e142fa7a8633",
+        "sha256": "16j95q58kkc69lfgpjkj76gw5sx8rcxwi3civm0mlfaxxyw9gzp6",
+        "type": "tarball",
+        "url": "https://github.com/nixos/nixpkgs/archive/a7e559a5504572008567383c3dc8e142fa7a8633.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "19.03": {
+        "branch": "nixos-19.03",
+        "description": "Nix Packages collection & NixOS",
+        "homepage": "",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "34c7eb7545d155cc5b6f499b23a7cb1c96ab4d59",
+        "sha256": "11z6ajj108fy2q5g8y4higlcaqncrbjm3dnv17pvif6avagw4mcb",
+        "type": "tarball",
+        "url": "https://github.com/nixos/nixpkgs/archive/34c7eb7545d155cc5b6f499b23a7cb1c96ab4d59.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "19.09": {
+        "branch": "nixos-19.09",
+        "description": "Nix Packages collection & NixOS",
+        "homepage": "",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "75f4ba05c63be3f147bcc2f7bd4ba1f029cedcb1",
+        "sha256": "157c64220lf825ll4c0cxsdwg7cxqdx4z559fdp7kpz0g6p8fhhr",
+        "type": "tarball",
+        "url": "https://github.com/nixos/nixpkgs/archive/75f4ba05c63be3f147bcc2f7bd4ba1f029cedcb1.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "20.03": {
+        "branch": "nixos-20.03",
+        "description": "Nix Packages collection & NixOS",
+        "homepage": "",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "1db42b7fe3878f3f5f7a4f2dc210772fd080e205",
+        "sha256": "05k9y9ki6jhaqdhycnidnk5zrdzsdammbk5lsmsbz249hjhhgcgh",
+        "type": "tarball",
+        "url": "https://github.com/nixos/nixpkgs/archive/1db42b7fe3878f3f5f7a4f2dc210772fd080e205.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "20.09": {
+        "branch": "nixos-20.09",
+        "description": "Nix Packages collection & NixOS",
+        "homepage": "",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "1c1f5649bb9c1b0d98637c8c365228f57126f361",
+        "sha256": "0f2nvdijyxfgl5kwyb4465pppd5vkhqxddx6v40k2s0z9jfhj0xl",
+        "type": "tarball",
+        "url": "https://github.com/nixos/nixpkgs/archive/1c1f5649bb9c1b0d98637c8c365228f57126f361.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "21.05": {
+        "branch": "nixos-21.05",
+        "description": "Nix Packages collection & NixOS",
+        "homepage": "",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "022caabb5f2265ad4006c1fa5b1ebe69fb0c3faf",
+        "sha256": "12q00nbd7fb812zchbcnmdg3pw45qhxm74hgpjmshc2dfmgkjh4n",
+        "type": "tarball",
+        "url": "https://github.com/nixos/nixpkgs/archive/022caabb5f2265ad4006c1fa5b1ebe69fb0c3faf.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "21.11": {
+        "branch": "nixos-21.11",
+        "description": "Nix Packages collection & NixOS",
+        "homepage": "",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "eabc38219184cc3e04a974fe31857d8e0eac098d",
+        "sha256": "04ffwp2gzq0hhz7siskw6qh9ys8ragp7285vi1zh8xjksxn1msc5",
+        "type": "tarball",
+        "url": "https://github.com/nixos/nixpkgs/archive/eabc38219184cc3e04a974fe31857d8e0eac098d.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "22.05": {
+        "branch": "nixos-22.05",
+        "description": "Nix Packages collection & NixOS",
+        "homepage": "",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "380be19fbd2d9079f677978361792cb25e8a3635",
+        "sha256": "154x9swf494mqwi4z8nbq2f0sp8pwp4fvx51lqzindjfbb9yxxv5",
+        "type": "tarball",
+        "url": "https://github.com/nixos/nixpkgs/archive/380be19fbd2d9079f677978361792cb25e8a3635.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "22.11": {
+        "branch": "nixos-22.11",
+        "description": "Nix Packages collection & NixOS",
+        "homepage": "",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "ea4c80b39be4c09702b0cb3b42eab59e2ba4f24b",
+        "sha256": "1xi53rlslcprybsvrmipm69ypd3g3hr7wkxvzc73ag8296yclyll",
+        "type": "tarball",
+        "url": "https://github.com/nixos/nixpkgs/archive/ea4c80b39be4c09702b0cb3b42eab59e2ba4f24b.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "23.05": {
+        "branch": "nixos-23.05",
+        "description": "Nix Packages collection & NixOS",
+        "homepage": "",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "70bdadeb94ffc8806c0570eb5c2695ad29f0e421",
+        "sha256": "05cbl1k193c9la9xhlz4y6y8ijpb2mkaqrab30zij6z4kqgclsrd",
+        "type": "tarball",
+        "url": "https://github.com/nixos/nixpkgs/archive/70bdadeb94ffc8806c0570eb5c2695ad29f0e421.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "23.11": {
+        "branch": "nixos-23.11",
+        "description": "Nix Packages collection & NixOS",
+        "homepage": "",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "219951b495fc2eac67b1456824cc1ec1fd2ee659",
+        "sha256": "065jy7qivlbdqmbvd7r9h97b23f21axmc4r7sqmq2h0j82rmymxv",
+        "type": "tarball",
+        "url": "https://github.com/nixos/nixpkgs/archive/219951b495fc2eac67b1456824cc1ec1fd2ee659.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    }
+}