nodePackages: patch node2nix for npm v7+ and switch to building package set with current nodejs

nixos/doc/manual/from_md/release-notes/rl-2211.section.xml

@@ -772,6 +772,18 @@
           which no longer has a downgrade path to releases 1.2 or older.
         </para>
       </listitem>
+      <listitem>
+        <para>
+          The <literal>nodePackages</literal> package set now defaults
+          to the LTS release in the <literal>nodejs</literal> package
+          again, instead of being pinned to
+          <literal>nodejs-14_x</literal>. Several updates to node2nix
+          have been made for compatibility with newer Node.js and npm
+          versions and a new <literal>postRebuild</literal> hook has
+          been added for packages to perform extra build steps before
+          the npm install step prunes dev dependencies.
+        </para>
+      </listitem>
     </itemizedlist>
   </section>
 </section>

nixos/doc/manual/release-notes/rl-2211.section.md

@@ -254,4 +254,6 @@ Available as [services.patroni](options.html#opt-services.patroni.enable).
 
 - The `nomad` package now defaults to 1.3, which no longer has a downgrade path to releases 1.2 or older.
 
+- The `nodePackages` package set now defaults to the LTS release in the `nodejs` package again, instead of being pinned to `nodejs-14_x`. Several updates to node2nix have been made for compatibility with newer Node.js and npm versions and a new `postRebuild` hook has been added for packages to perform extra build steps before the npm install step prunes dev dependencies.
+
 <!-- To avoid merge conflicts, consider adding your item at an arbitrary place in the list instead. -->

pkgs/applications/editors/vim/plugins/markdown-preview-nvim/package.json

@@ -1,18 +1,48 @@
 {
-  "name": "markdown-preview-vim",
-  "version": "0.0.1",
+  "name": "markdown-preview",
+  "version": "0.0.10",
   "description": "markdown preview plugin for (neo)vim",
-  "bin": "./index.js",
-  "repository": "https://github.com/iamcco/markdown-preview.vim.git",
+  "bin": "./app/server.js",
+  "repository": "https://github.com/iamcco/markdown-preview.nvim.git",
   "author": "年糕小豆汤 <[email protected]>",
   "license": "MIT",
   "private": true,
+  "scripts": {
+    "watch": "tsc -w -p ./",
+    "build-app": "cd app && rm -rf ./.next && next build && next export",
+    "build-lib": "tsc -p ./",
+    "build": "tsc -p ./ && cd app && rm -rf ./.next && next build && next export && yarn && pkg --targets node16-linux-x64,node16-macos-x64,node16-win-x64 --out-path ./bin . && rm -rf ./node_modules ./.next"
+  },
   "dependencies": {
-    "@chemzqm/neovim": "5.7.9",
-    "log4js": "3.0.6",
-    "neovim": "4.2.1",
-    "socket.io": "2.1.1",
-    "tslib": "1.9.3",
-    "vim-node-rpc": "0.1.24"
+    "@chemzqm/neovim": "^5.7.9",
+    "chart.js": "^2.7.3",
+    "highlight.js": "^10.4.1",
+    "log4js": "^6.4.0",
+    "markdown-it": "^12.3.2",
+    "markdown-it-anchor": "^5.2.4",
+    "markdown-it-deflist": "^2.0.3",
+    "markdown-it-emoji": "^1.4.0",
+    "markdown-it-footnote": "^3.0.1",
+    "markdown-it-task-lists": "^2.1.1",
+    "markdown-it-toc-done-right": "^4.0.1",
+    "md-it-meta": "^0.0.2",
+    "msgpack-lite": "^0.1.26",
+    "next": "^7.0.2",
+    "next-routes": "^1.4.2",
+    "plantuml-encoder": "^1.4.0",
+    "react": "^16.5.2",
+    "react-dom": "^16.5.2",
+    "socket.io": "^2.1.1",
+    "socket.io-client": "^2.1.1"
+  },
+  "devDependencies": {
+    "@types/node": "16",
+    "pkg": "^5.6.0",
+    "prettier": "^2.6.2",
+    "tslint": "^6.1.3",
+    "tslint-config-prettier": "^1.18.0",
+    "tslint-plugin-prettier": "^2.3.0",
+    "typescript": "^4.6.4",
+    "yuuko-tsconfig": "^1.0.0"
   }
 }

pkgs/applications/editors/vim/plugins/overrides.nix

@@ -553,8 +553,7 @@ self: super: {
       })
     ];
     postInstall = ''
-      # The node package name is `*-vim` not `*-nvim`.
-      ln -s ${nodeDep}/lib/node_modules/markdown-preview-vim/node_modules $out/app
+      ln -s ${nodeDep}/lib/node_modules/markdown-preview/node_modules $out/app
     '';
 
     nativeBuildInputs = [ nodejs ];

pkgs/applications/editors/vscode/extensions/rust-analyzer/default.nix

@@ -48,7 +48,7 @@ let
     ];
 
     # Follows https://github.com/rust-lang/rust-analyzer/blob/41949748a6123fd6061eb984a47f4fe780525e63/xtask/src/dist.rs#L39-L65
-    postInstall = ''
+    postRebuild = ''
       jq '
         .version = $ENV.version |
         .releaseTag = $ENV.releaseTag |

pkgs/applications/networking/instant-messengers/deltachat-desktop/default.nix

@@ -73,16 +73,13 @@ in nodePackages.deltachat-desktop.override rec {
   USE_SYSTEM_LIBDELTACHAT = "true";
   VERSION_INFO_GIT_REF = src.rev;
 
-  preRebuild = ''
-    substituteInPlace package.json \
-      --replace "node ./bin/check-nodejs-version.js" true
-  '';
-
-  postInstall = ''
+  postRebuild = ''
     rm -r node_modules/deltachat-node/node/prebuilds
 
     npm run build4production
+  '';
 
+  postInstall = ''
     npm prune --production
 
     install -D $out/lib/node_modules/deltachat-desktop/build/icon.png \

pkgs/applications/video/epgstation/default.nix

@@ -6,7 +6,6 @@
 , makeWrapper
 , bash
 , nodejs
-, nodePackages
 , gzip
 , jq
 , yq
@@ -23,18 +22,17 @@ let
     sha256 = "K1cAvmqWEfS6EY4MKAtjXb388XLYHtouxNM70PWgFig=";
   };
 
-  client = nodePackages.epgstation-client.override (drv: {
-    # FIXME: remove this option if possible
-    #
-    # Unsetting this option resulted NPM attempting to re-download packages.
-    dontNpmInstall = true;
+  client = nodejs.pkgs.epgstation-client.override (drv: {
+    # This is set to false to keep devDependencies at build time. Build time
+    # dependencies are pruned afterwards.
+    production = false;
 
     meta = drv.meta // {
       inherit (nodejs.meta) platforms;
     };
   });
 
-  server = nodePackages.epgstation.override (drv: {
+  server = nodejs.pkgs.epgstation.override (drv: {
     inherit src;
 
     # This is set to false to keep devDependencies at build time. Build time
@@ -47,6 +45,9 @@ let
     ];
 
     preRebuild = ''
+      # Fix for OpenSSL compat with newer Node.js
+      export NODE_OPTIONS=--openssl-legacy-provider
+
       # Fix for not being able to connect to mysql using domain sockets.
       patch -p1 < ${./use-mysql-over-domain-socket.patch}
 
@@ -64,8 +65,8 @@ let
 
       pushd $out/lib/node_modules/epgstation
 
-      cp -r ${client}/lib/node_modules/epgstation-client/node_modules client/node_modules
-      chmod -R u+w client/node_modules
+      cp -r ${client}/lib/node_modules/epgstation-client/{package-lock.json,node_modules} client/
+      chmod -R u+w client/{package-lock.json,node_modules}
 
       npm run build
 

pkgs/applications/video/epgstation/package.json

@@ -35,7 +35,7 @@
     "reflect-metadata": "0.1.13",
     "socket.io": "4.3.1",
     "source-map-support": "0.5.20",
-    "sqlite3": "5.0.2",
+    "sqlite3": "5.0.11",
     "swagger-ui-dist": "3.52.5",
     "typeorm": "0.2.38",
     "url-join": "4.0.1",

pkgs/applications/video/epgstation/update.nix

@@ -6,6 +6,7 @@
 , writers
 , jq
 , yq
+, gnused
 }:
 
 let
@@ -45,6 +46,9 @@ in writers.writeBash "update-epgstation" ''
     "$SRC/client/package.json" \
     > client/package.json
 
+  # Fix issue with old sqlite3 version pinned that depends on very old node-gyp 3.x
+  ${gnused}/bin/sed -i -e 's/"sqlite3":\s*"5.0.[0-9]\+"/"sqlite3": "5.0.11"/' package.json
+
   # Regenerate node packages to update the pre-overriden epgstation derivation.
   # This must come *after* package.json has been regenerated.
   pushd ../../../development/node-packages

pkgs/development/node-packages/composition.nix

@@ -2,7 +2,7 @@
 
 {pkgs ? import <nixpkgs> {
     inherit system;
-  }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-14_x"}:
+  }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-18_x"}:
 
 let
   nodeEnv = import ./node-env.nix {

pkgs/development/node-packages/generate.sh

@@ -13,7 +13,7 @@ rm -f ./node-env.nix
     -i node-packages.json \
     -o node-packages.nix \
     -c composition.nix \
-    --pkg-name nodejs-14_x
+    --pkg-name nodejs-18_x
 
 # using --no-out-link in nix-build argument would cause the
 # gc to run before the script finishes

pkgs/development/node-packages/node-env.nix

@@ -165,7 +165,11 @@ let
           if(process.argv[2] == "development") {
               replaceDependencies(packageObj.devDependencies);
           }
+          else {
+              packageObj.devDependencies = {};
+          }
           replaceDependencies(packageObj.optionalDependencies);
+          replaceDependencies(packageObj.peerDependencies);
 
           /* Write the fixed package.json file */
           fs.writeFileSync("package.json", JSON.stringify(packageObj, null, 2));
@@ -270,7 +274,7 @@ let
 
   # Reconstructs a package-lock file from the node_modules/ folder structure and package.json files with dummy sha1 hashes
   reconstructPackageLock = writeTextFile {
-    name = "addintegrityfields.js";
+    name = "reconstructpackagelock.js";
     text = ''
       var fs = require('fs');
       var path = require('path');
@@ -280,25 +284,43 @@ let
       var lockObj = {
           name: packageObj.name,
           version: packageObj.version,
-          lockfileVersion: 1,
+          lockfileVersion: 2,
           requires: true,
+          packages: {
+              "": {
+                  name: packageObj.name,
+                  version: packageObj.version,
+                  license: packageObj.license,
+                  bin: packageObj.bin,
+                  dependencies: packageObj.dependencies,
+                  engines: packageObj.engines,
+                  optionalDependencies: packageObj.optionalDependencies
+              }
+          },
           dependencies: {}
       };
 
-      function augmentPackageJSON(filePath, dependencies) {
+      function augmentPackageJSON(filePath, packages, dependencies) {
           var packageJSON = path.join(filePath, "package.json");
           if(fs.existsSync(packageJSON)) {
               var packageObj = JSON.parse(fs.readFileSync(packageJSON));
+              packages[filePath] = {
+                  version: packageObj.version,
+                  integrity: "sha1-000000000000000000000000000=",
+                  dependencies: packageObj.dependencies,
+                  engines: packageObj.engines,
+                  optionalDependencies: packageObj.optionalDependencies
+              };
               dependencies[packageObj.name] = {
                   version: packageObj.version,
                   integrity: "sha1-000000000000000000000000000=",
                   dependencies: {}
               };
-              processDependencies(path.join(filePath, "node_modules"), dependencies[packageObj.name].dependencies);
+              processDependencies(path.join(filePath, "node_modules"), packages, dependencies[packageObj.name].dependencies);
           }
       }
 
-      function processDependencies(dir, dependencies) {
+      function processDependencies(dir, packages, dependencies) {
           if(fs.existsSync(dir)) {
               var files = fs.readdirSync(dir);
 
@@ -314,23 +336,84 @@ let
                           pkgFiles.forEach(function(entry) {
                               if(stats.isDirectory()) {
                                   var pkgFilePath = path.join(filePath, entry);
-                                  augmentPackageJSON(pkgFilePath, dependencies);
+                                  augmentPackageJSON(pkgFilePath, packages, dependencies);
                               }
                           });
                       } else {
-                          augmentPackageJSON(filePath, dependencies);
+                          augmentPackageJSON(filePath, packages, dependencies);
                       }
                   }
               });
           }
       }
 
-      processDependencies("node_modules", lockObj.dependencies);
+      processDependencies("node_modules", lockObj.packages, lockObj.dependencies);
 
       fs.writeFileSync("package-lock.json", JSON.stringify(lockObj, null, 2));
     '';
   };
 
+  # Script that links bins defined in package.json to the node_modules bin directory
+  # NPM does not do this for top-level packages itself anymore as of v7
+  linkBinsScript = writeTextFile {
+    name = "linkbins.js";
+    text = ''
+      var fs = require('fs');
+      var path = require('path');
+
+      var packageObj = JSON.parse(fs.readFileSync("package.json"));
+
+      var nodeModules = Array(packageObj.name.split("/").length).fill("..").join(path.sep);
+
+      if(packageObj.bin !== undefined) {
+          fs.mkdirSync(path.join(nodeModules, ".bin"))
+
+          if(typeof packageObj.bin == "object") {
+              Object.keys(packageObj.bin).forEach(function(exe) {
+                  if(fs.existsSync(packageObj.bin[exe])) {
+                      console.log("linking bin '" + exe + "'");
+                      fs.symlinkSync(
+                          path.join("..", packageObj.name, packageObj.bin[exe]),
+                          path.join(nodeModules, ".bin", exe)
+                      );
+                  }
+                  else {
+                      console.log("skipping non-existent bin '" + exe + "'");
+                  }
+              })
+          }
+          else {
+              if(fs.existsSync(packageObj.bin)) {
+                  console.log("linking bin '" + packageObj.bin + "'");
+                  fs.symlinkSync(
+                      path.join("..", packageObj.name, packageObj.bin),
+                      path.join(nodeModules, ".bin", packageObj.name.split("/").pop())
+                  );
+              }
+              else {
+                  console.log("skipping non-existent bin '" + packageObj.bin + "'");
+              }
+          }
+      }
+      else if(packageObj.directories !== undefined && packageObj.directories.bin !== undefined) {
+          fs.mkdirSync(path.join(nodeModules, ".bin"))
+
+          fs.readdirSync(packageObj.directories.bin).forEach(function(exe) {
+              if(fs.existsSync(path.join(packageObj.directories.bin, exe))) {
+                  console.log("linking bin '" + exe + "'");
+                  fs.symlinkSync(
+                      path.join("..", packageObj.name, packageObj.directories.bin, exe),
+                      path.join(nodeModules, ".bin", exe)
+                  );
+              }
+              else {
+                  console.log("skipping non-existent bin '" + exe + "'");
+              }
+          })
+      }
+    '';
+  };
+
   prepareAndInvokeNPM = {packageName, bypassCache, reconstructLock, npmFlags, production}:
     let
       forceOfflineFlag = if bypassCache then "--offline" else "--registry http://www.example.com";
@@ -377,13 +460,18 @@ let
 
         npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${lib.optionalString production "--production"} rebuild
 
+        runHook postRebuild
+
         if [ "''${dontNpmInstall-}" != "1" ]
         then
             # NPM tries to download packages even when they already exist if npm-shrinkwrap is used.
             rm -f npm-shrinkwrap.json
 
-            npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${lib.optionalString production "--production"} install
+            npm ${forceOfflineFlag} --nodedir=${nodeSources} --no-bin-links --ignore-scripts ${npmFlags} ${lib.optionalString production "--production"} install
         fi
+
+        # Link executables defined in package.json
+        node ${linkBinsScript}
     '';
 
   # Builds and composes an NPM package including all its dependencies