Add Nixpkgs invocation helper modules

doc/using/configuration.chapter.md

@@ -191,6 +191,13 @@ list-id: configuration-variable-list
 source: ../config-options.json
 ```
 
+## Calling Nixpkgs from the Module System {#sec-invoke-nixpkgs-by-module}
+
+For [module system](#module-system) application authors, Nixpkgs provides a reusable module for the purpose of invoking Nixpkgs. Most users should not have to be aware of this. The documentation for its options is meant to be re-exposed in the application's documentation and is therefore not repeated here.
+
+Applications can import the implementation file [`"${nixpkgs}/pkgs/top-level/module/module.nix"`](https://github.com/NixOS/nixpkgs/blob/master/pkgs/top-level/module/module.nix).
+
+The flake attribute (experimental) for this module is `.modules.generic.nixpkgs`.
 
 ## Declarative Package Management {#sec-declarative-package-management}
 

flake.nix

@@ -12,6 +12,20 @@
       lib = import ./lib;
 
       forAllSystems = lib.genAttrs lib.systems.flakeExposed;
+
+      pkgsMemoizer = { pkgs, inputsSet, ... }:
+        let system = lib.systems.toLosslessStringMaybe inputsSet.hostPlatform.value.system;
+        in
+        if lib.attrNames inputsSet == [ "hostPlatform" ] && system != null
+        then
+          # As only hostPlatform matters in this invocation, we can save
+          # memory by sharing the Nixpkgs invocation.
+          # `pkgs` is never evaluated.
+          self.legacyPackages.${system}
+        else
+          # We let the module invoke Nixpkgs as usual.
+          pkgs;
+
     in
     {
       lib = lib.extend (final: prev: {
@@ -58,18 +72,28 @@
       nixosModules = {
         notDetected = ./nixos/modules/installer/scan/not-detected.nix;
 
-        /*
-          Make the `nixpkgs.*` configuration read-only. Guarantees that `pkgs`
-          is the way you initialize it.
+        # See nixos/doc/manual/nixos-optional-modules.md
+        # TODO: online manual link
+        readOnlyPkgs = ./nixos/modules/misc/nixpkgs/read-only.nix;
 
-          Example:
+        # See nixos/doc/manual/nixos-optional-modules.md
+        # TODO: online manual link
+        noLegacyPkgs = {
+          imports = [ ./nixos/modules/misc/nixpkgs/no-legacy.nix ];
+          nixpkgs._memoize = pkgsMemoizer;
+        };
+      };
 
-              {
-                imports = [ nixpkgs.nixosModules.readOnlyPkgs ];
-                nixpkgs.pkgs = nixpkgs.legacyPackages.x86_64-linux;
-              }
-        */
-        readOnlyPkgs = ./nixos/modules/misc/nixpkgs/read-only.nix;
+      modules = {
+        /* Modules that are not specific to any application of the module system. */
+        generic = {
+          nixpkgs = {
+            imports = [
+              ./pkgs/top-level/module/module.nix
+            ];
+            config._memoize = pkgsMemoizer;
+          };
+        };
       };
     };
 }

nixos/doc/manual/default.nix

@@ -35,18 +35,14 @@ let
     };
   };
 
-  nixos-lib = import ../../lib { };
+  nixos-lib = import ../../lib { featureFlags = {
+    # We use a minimal module list to evaluate the docs of the extra modules.
+    # This is significantly faster than loading all the modules, and we can pull
+    # this off because we don't require any dependencies to be loaded.
+    minimalModules = { }; };
+  };
 
-  testOptionsDoc = let
-      eval = nixos-lib.evalTest {
-        # Avoid evaluating a NixOS config prototype.
-        config.node.type = lib.types.deferredModule;
-        options._module.args = lib.mkOption { internal = true; };
-      };
-    in buildPackages.nixosOptionsDoc {
-      inherit (eval) options;
-      inherit revision;
-      transformOptions = opt: opt // {
+  cleanupLocations = opt: opt // {
         # Clean up declaration sites to not refer to the NixOS source tree.
         declarations =
           map
@@ -58,11 +54,50 @@ let
               else decl)
             opt.declarations;
       };
+
+  testOptionsDoc = let
+      eval = nixos-lib.evalTest {
+        # Avoid evaluating a NixOS config prototype.
+        config.node.type = lib.types.deferredModule;
+        options._module.args = lib.mkOption { internal = true; };
+      };
+    in buildPackages.nixosOptionsDoc {
+      inherit (eval) options;
+      inherit revision;
+      transformOptions = cleanupLocations;
       documentType = "none";
       variablelistId = "test-options-list";
       optionIdPrefix = "test-opt-";
     };
 
+  optionalDocs = lib.mapAttrs (name: module:
+    let
+      # This is quite simple for now, but may need stubs for more complex modules.
+      eval = nixos-lib.evalModules {
+        modules = [
+          module
+          { options._module.args = lib.mkOption { internal = true; }; }
+        ];
+      };
+    in
+      buildPackages.nixosOptionsDoc {
+        inherit (eval) options;
+        inherit revision;
+        transformOptions = cleanupLocations;
+        # These are for direct to docbook generation, which we don't use here.
+        documentType = throw "documentType not set";
+        variablelistId = throw "variablelistId not set";
+        optionIdPrefix = throw "optionIdPrefix not set";
+      }
+  ) {
+    # NOTE: These don't have to be paths. If a module needs dependencies to be loaded
+    #       for doc rendering, do something like
+    #           newModule = { imports = [ ../../modules/new.nix ../../modules/dep.nix ]; }
+    #       or import it transitively.
+    readOnlyPkgs = ../../modules/misc/nixpkgs/read-only.nix;
+    noLegacyPkgs = ../../modules/misc/nixpkgs/no-legacy.nix;
+  };
+
   prepareManualFromMD = ''
     cp -r --no-preserve=all $inputs/* .
 
@@ -76,6 +111,14 @@ let
       --replace \
         '@NIXOS_OPTIONS_JSON@' \
         ${optionsDoc.optionsJSON}/share/doc/nixos/options.json
+    substituteInPlace ./nixos-optional-modules.md \
+      --replace \
+        '@OPTIONS_JSON_noLegacyPkgs@' \
+        ${optionalDocs.noLegacyPkgs.optionsJSON}/share/doc/nixos/options.json \
+      --replace \
+        '@OPTIONS_JSON_readOnlyPkgs@' \
+        ${optionalDocs.readOnlyPkgs.optionsJSON}/share/doc/nixos/options.json \
+        ;
     substituteInPlace ./development/writing-nixos-tests.section.md \
       --replace \
         '@NIXOS_TEST_OPTIONS_JSON@' \

nixos/doc/manual/manual.md

@@ -51,6 +51,10 @@ contributing-to-this-manual.chapter.md
 nixos-options.md
 ```
 
+```{=include=} appendix html:into-file=//optional-modules.html
+nixos-optional-modules.md
+```
+
 ```{=include=} appendix html:into-file=//release-notes.html
 release-notes/release-notes.md
 ```

nixos/doc/manual/nixos-optional-modules.md

@@ -0,0 +1,53 @@
+# Optional Modules {#ch-optional-modules}
+
+## `noLegacyPkgs` {#sec-noLegacyPkgs}
+
+This module reimplements `nixpkgs.*` without legacy options such as `nixpkgs.localSystem`.
+
+When imported from the flake (experimental), this module will reuse `legacyPackages` when only `hostPlatform` is specified.
+
+This module is ignored when [`readOnlyPkgs`](#sec-readOnlyPkgs) is imported.
+
+Example use with a flake (experimental):
+
+```nix
+inputs.nixpkgs.lib.nixosSystem {
+  modules = [
+    inputs.nixpkgs.nixosModules.noLegacyPkgs
+    ./configuration.nix
+  ];
+}
+```
+
+```{=include=} options
+id-prefix: module-noLegacyPkgs-opt-
+list-id: module-noLegacyPkgs-options
+source: @OPTIONS_JSON_noLegacyPkgs@
+```
+
+## `readOnlyPkgs` {#sec-readOnlyPkgs}
+
+This module ensures that the `pkgs` module argument is as specified, manually.
+
+The usual `nixpkgs.*` options are replaced by read-only options for compatibility. These are not listed here. NixOS modules should
+get their information from the `pkgs` module argument and ignore `nixpkgs.*`.
+
+Example use with a flake (experimental):
+
+```nix
+inputs.nixpkgs.lib.nixosSystem {
+  modules = [
+    inputs.nixpkgs.nixosModules.readOnlyPkgs
+    ./configuration.nix
+    {
+      nixpkgs.pkgs = inputs.nixpkgs.legacyPackages.x86_64-linux;
+    }
+  ];
+}
+```
+
+```{=include=} options
+id-prefix: module-readOnlyPkgs-opt-
+list-id: module-readOnlyPkgs-options
+source: @OPTIONS_JSON_readOnlyPkgs@
+```

nixos/doc/manual/release-notes/rl-2311.section.md

@@ -82,6 +82,23 @@
 
 - A new option was added to the virtualisation module that enables specifying explicitly named network interfaces in QEMU VMs. The existing `virtualisation.vlans` is still supported for cases where the name of the network interface is irrelevant.
 
+- The flake now exposes a module [`modules.generic.nixpkgs`](https://nixos.org/manual/nixpkgs/unstable/#sec-invoke-nixpkgs-by-module) for configuring and invoking Nixpkgs, to be used in new or existing module system applications.
+  It is more efficient than a simple reimplementation, and it carries none of the NixOS legacy options.
+
+- Importable modules have been added for flakes users to optimize `pkgs` in NixOS.
+
+  - To reuse the `legacyPackages.*` instantiation of Nixpkgs *if applicable*, use.
+    ```nix
+    imports = [ nixpkgs.nixosModules.noLegacyPkgs ];
+    ```
+    Here, legacy refers to old `nixpkgs.*` options.
+
+  - If you want to be sure that a shared `pkgs` instance is reused, you may invoke Nixpkgs yourself and use `readOnlyPkgs`, which makes sure the other `nixpkgs.*` options are unset.
+    ```nix
+    imports = [ nixpkgs.nixosModules.readOnlyPkgs ];
+    nixpkgs.pkgs = ...;
+    ```
+
 - DocBook option documentation is no longer supported, all module documentation now uses markdown.
 
 - `services.fail2ban.jails` can now be configured with attribute sets defining settings and filters instead of lines. The stringed options `daemonConfig` and `extraSettings` have respectively been replaced by `daemonSettings` and `jails.DEFAULT.settings` which use attribute sets.

nixos/modules/misc/nixpkgs/no-legacy.nix

@@ -0,0 +1,39 @@
+/*
+  A clean, [RFC 0078] style implementation of the NixOS `nixpkgs.*` module, which
+  is free of legacy options (as of 23.05).
+
+  [RFC 0078]: https://github.com/NixOS/rfcs/pull/78
+*/
+
+{ config, lib, ... }:
+
+{
+  disabledModules = [
+    # This replaces the traditional nixpkgs module
+    ../nixpkgs.nix
+  ];
+  options = {
+    nixpkgs = lib.mkOption {
+      description = lib.mdDoc ''
+        Options that configure the `pkgs` module argument.
+      '';
+      example = lib.literalExpression ''
+        nixpkgs = {
+          config.allowUnfree = true;
+          overlays = [
+            (final: prev: {
+              # Patch nixpkgs here
+            })
+          ];
+        };
+      '';
+      type = lib.types.submoduleWith {
+        shorthandOnlyDefinesConfig = true;
+        modules = [ ../../../../pkgs/top-level/module/module.nix ];
+      };
+    };
+  };
+  config = {
+    _module.args.pkgs = config.nixpkgs.pkgs;
+  };
+}

nixos/modules/misc/nixpkgs/read-only.nix

@@ -19,6 +19,7 @@ in
 {
   disabledModules = [
     ../nixpkgs.nix
+    ./no-legacy.nix
   ];
   options = {
     nixpkgs = {

nixos/modules/misc/nixpkgs/test.nix

@@ -68,6 +68,11 @@ let
     nixpkgs.buildPlatform = "foo-linux"; # do in pkgs instead!
   };
 
+  noLegacy = evalMinimalConfig {
+    imports = [ ./no-legacy.nix ];
+    nixpkgs.hostPlatform = "aarch64-linux";
+  };
+
   throws = x: ! (builtins.tryEval x).success;
 
 in
@@ -124,5 +129,11 @@ lib.recurseIntoAttrs {
     assert !readOnly.options.nixpkgs?localSystem;
     assert !readOnly.options.nixpkgs?crossSystem;
 
+    assert noLegacy._module.args.pkgs.stdenv.hostPlatform.system == "aarch64-linux";
+    assert noLegacy._module.args.pkgs.stdenv.buildPlatform.system == "aarch64-linux";
+    assert lib.systems.equals noLegacy._module.args.pkgs.stdenv.hostPlatform withHost._module.args.pkgs.stdenv.hostPlatform;
+    assert lib.systems.equals noLegacy._module.args.pkgs.stdenv.buildPlatform withHost._module.args.pkgs.stdenv.buildPlatform;
+    assert noLegacy._module.args.pkgs.hello == withHost._module.args.pkgs.hello;
+
     pkgs.emptyFile;
 }

nixos/modules/virtualisation/qemu-vm.nix

@@ -630,7 +630,7 @@ in
       };
 
     virtualisation.host.pkgs = mkOption {
-      type = options.nixpkgs.pkgs.type;
+      type = types.pkgs;
       default = pkgs;
       defaultText = literalExpression "pkgs";
       example = literalExpression ''

pkgs/test/default.nix

@@ -101,4 +101,10 @@ with pkgs;
   };
 
   pkgs-lib = recurseIntoAttrs (import ../pkgs-lib/tests { inherit pkgs; });
+
+  top-level = recurseIntoAttrs {
+    module = recurseIntoAttrs (
+      (callPackage ../top-level/module/tests.nix { }).tests
+    );
+  };
 }