module system: internalise prefix-stripping logic

Add an internal `_module.sources` option which specifies a list of
sources to be used by `lib.strings.lookupPrefix` to render paths in
option values and declarations.
The sources always include the `nixpkgsSource`, which links to the
manual for `lib.trivial.manualRevision`.

Implement this logic in a new `lib.optionsToDocTemplate` function which
deprecates `lib.optionAttrSetToDocList`. The returned option
declarations are now attrsets as returned by `lookupPrefix`.
If `decl.source` is set, then the declaration is rendered using angle
bracket syntax: `<${decl.source}/${decl.path}>`. Other consumers are
free to treat the source name as they intend.

Deprecate `documentation.nixos.extraModuleSources`, converting it to
`_module.sources` with `name` set to `_unknown`.

We now call `eval-cacheable-options.nix` with a single filtered nixpkgs
source, which is easier to maintain than a bunch of paths and allows
things to work more smoothly (mainly `lib.trivial.release` can access
../.version, and `lib.nixpkgsSource` works as intended so we don't
need to special-case anything).

doc/doc-support/default.nix

@@ -32,18 +32,6 @@ let
   optionsDoc = pkgs.nixosOptionsDoc {
     inherit (pkgs.lib.evalModules { modules = [ ../../pkgs/top-level/config.nix ]; }) options;
     documentType = "none";
-    transformOptions = opt:
-      opt // {
-        declarations =
-          map
-            (decl:
-              if hasPrefix (toString ../..) (toString decl)
-              then
-                let subpath = removePrefix "/" (removePrefix (toString ../..) (toString decl));
-                in { url = "https://github.com/NixOS/nixpkgs/blob/master/${subpath}"; name = subpath; }
-              else decl)
-            opt.declarations;
-        };
   };
 
 in pkgs.runCommand "doc-support" {}

lib/default.nix

@@ -131,7 +131,7 @@ let
     inherit (self.options) isOption mkEnableOption mkSinkUndeclaredOptions
       mergeDefaultOption mergeOneOption mergeEqualOption mergeUniqueOption
       getValues getFiles
-      optionAttrSetToDocList optionAttrSetToDocList'
+      optionAttrSetToDocList optionsToDocTemplate
       scrubOptionValue literalExpression literalExample literalDocBook
       showOption showOptionWithDefLocs showFiles
       unknownModule mkOption mkPackageOption

lib/modules.nix

@@ -137,10 +137,10 @@ rec {
       # When extended with extendModules or moduleType, a fresh instance of
       # this module is used, to avoid conflicts and allow chaining of
       # extendModules.
-      internalModule = rec {
-        _file = "lib/modules.nix";
+      internalModule = {
+        _file = ./modules.nix;
 
-        key = _file;
+        key = "lib/modules.nix";
 
         options = {
           _module.args = mkOption {
@@ -238,6 +238,32 @@ rec {
               within a configuration, but can be used in module imports.
             '';
           };
+
+          _module.sources = mkOption {
+            type = with types; let
+              # Using a submodule here results in infinite recursion, unsurprisingly
+              sourceType = mkOptionType {
+                name = "source";
+                description = "module source accepted by lib.strings.lookupPrefix";
+                descriptionClass = "composite";
+                check = source: isAttrs source
+                  && source ? root && path.check source.root
+                  && source ? name && str.check source.name
+                  && (source ? mkUrl -> isFunction source.mkUrl);
+                merge = lib.options.mergeOneOption;
+              };
+            in listOf sourceType;
+            default = [ ];
+            internal = true;
+            description = lib.mdDoc ''
+              A list of module sources, in the format accepted by `lib.strings.lookupPrefix`.
+              These are used by `lib.optionsToDocTemplate` for rendering store paths as URLs in documentation.
+            '';
+            example = lib.literalExpression ''
+              # e.g. with options from modules in ''${pkgs.foo}/nix:
+              [ { name = "foo"; root = pkgs.foo; mkUrl = m: "https://example.com/foo/" + m; ]
+            '';
+          };
         };
 
         config = {
@@ -246,6 +272,7 @@ rec {
             moduleType = type;
           };
           _module.specialArgs = specialArgs;
+          _module.sources = mkAfter [ lib.nixpkgsSource ];
         };
       };
 
@@ -646,7 +673,7 @@ rec {
      'loc' is the list of attribute names where the option is located.
 
      'opts' is a list of modules.  Each module has an options attribute which
-     correspond to the definition of 'loc' in 'opt.file'. */
+     correspond to the definition of 'loc' in 'opt._file'. */
   mergeOptionDecls =
    loc: opts:
     foldl' (res: opt:
@@ -673,7 +700,12 @@ rec {
             if getSubModules != null then map (setDefaultModuleLocation opt._file) getSubModules ++ res.options
             else res.options;
         in opt.options // res //
-          { declarations = res.declarations ++ [opt._file];
+          { declarations =
+              # Ensure the "main" declaration (the one with the default if there's one,
+              # description otherwise) ends up first in the list.
+              if opt.options ? default || (! (res ? default) && opt.options ? description)
+              then [opt._file] ++ res.declarations
+              else res.declarations ++ [opt._file];
             options = submodules;
           } // typeSet
     ) { inherit loc; declarations = []; options = []; } opts;
@@ -901,7 +933,7 @@ rec {
   mkBefore = mkOrder 500;
   mkAfter = mkOrder 1500;
 
-  # The default priority for things that don't have a priority specified.
+  # The default override priority for things that don't have a priority specified.
   defaultPriority = 100;
 
   # Convenient property used to transfer all definitions and their

lib/options.nix

@@ -214,38 +214,64 @@ rec {
   */
   getFiles = map (x: x.file);
 
-  # Generate documentation template from the list of option declaration like
-  # the set generated with filterOptionSets.
-  optionAttrSetToDocList = optionAttrSetToDocList' [];
+  /* Generates a flat list of options suitable for documentation generation from
+     an `options` attrset produced by `evalModules`.
 
-  optionAttrSetToDocList' = _: options:
-    concatMap (opt:
-      let
-        docOption = rec {
-          loc = opt.loc;
-          name = showOption opt.loc;
-          description = opt.description or null;
-          declarations = filter (x: x != unknownModule) opt.declarations;
-          internal = opt.internal or false;
-          visible =
-            if (opt?visible && opt.visible == "shallow")
-            then true
-            else opt.visible or true;
-          readOnly = opt.readOnly or false;
-          type = opt.type.description or "unspecified";
-        }
-        // optionalAttrs (opt ? example) { example = renderOptionValue opt.example; }
-        // optionalAttrs (opt ? default) { default = renderOptionValue (opt.defaultText or opt.default); }
-        // optionalAttrs (opt ? relatedPackages && opt.relatedPackages != null) { inherit (opt) relatedPackages; };
-
-        subOptions =
-          let ss = opt.type.getSubOptions opt.loc;
-          in if ss != {} then optionAttrSetToDocList' opt.loc ss else [];
-        subOptionsVisible = docOption.visible && opt.visible or null != "shallow";
-      in
-        # To find infinite recursion in NixOS option docs:
-        # builtins.trace opt.loc
-        [ docOption ] ++ optionals subOptionsVisible subOptions) (collect isOption options);
+     The returned options obey the following specification:
+
+     - `type` is a string
+     - `default` and `example` are {_type, text} attrsets as returned by
+       `literalExpression`, `literalMD`, etc.
+     - `declarations` is a list of {path, source?, url?} attrsets as returned by
+       `lib.strings.lookupPrefix`
+  */
+  optionsToDocTemplate =
+    { options
+    # Additional transformation to apply to each option.
+    , transformOptions ? lib.id
+    }: let
+      moduleSources = options._module.sources.value or [];
+      mkDeclaration = lib.strings.lookupPrefix moduleSources;
+      renderOptionValue = v:
+        if v ? _type && v ? text then v
+        else literalExpression (lib.generators.toPretty {
+          multiline = true;
+          allowPrettyValues = true;
+          searchPath = moduleSources;
+        } v);
+      go = opts:
+        lib.flip concatMap (collect isOption opts) (opt: let
+          docOption = rec {
+            loc = opt.loc;
+            name = showOption opt.loc;
+            description = opt.description or null;
+            declarations = map mkDeclaration (filter (x: x != unknownModule) opt.declarations);
+            internal = opt.internal or false;
+            visible =
+              (opt?visible && opt.visible == "shallow")
+              || opt.visible or true;
+            readOnly = opt.readOnly or false;
+            type = opt.type.description or "unspecified";
+          }
+          // optionalAttrs (opt ? example) { example = renderOptionValue opt.example; }
+          // optionalAttrs (opt ? default) { default = renderOptionValue (opt.defaultText or opt.default); }
+          // optionalAttrs (opt ? relatedPackages && opt.relatedPackages != null) { inherit (opt) relatedPackages; };
+
+          subOptions =
+            let ss = opt.type.getSubOptions opt.loc;
+            in if ss != {} then go ss else [];
+          subOptionsVisible = docOption.visible && opt.visible or null != "shallow";
+        in
+          # To find infinite recursion in NixOS option docs:
+          # builtins.trace opt.loc
+          [ (transformOptions docOption) ] ++ optionals subOptionsVisible subOptions);
+    in go options;
+
+  /* For backwards compatibility. */
+  optionAttrSetToDocList = options: optionsToDocTemplate {
+    inherit options;
+    transformOptions = opt: opt // { declarations = map (d: d.path) opt.declarations; };
+  };
 
 
   /* This function recursively removes all derivation attributes from
@@ -256,7 +282,7 @@ rec {
      (on the order of megabytes) and is not actually used by the
      manual generator.
 
-     This function was made obsolete by renderOptionValue and is kept for
+     This function was made obsolete by optionsToDocTemplate and is kept for
      compatibility with out-of-tree code.
   */
   scrubOptionValue = x:
@@ -267,17 +293,6 @@ rec {
     else x;
 
 
-  /* Ensures that the given option value (default or example) is a `_type`d string
-     by rendering Nix values to `literalExpression`s.
-  */
-  renderOptionValue = v:
-    if v ? _type && v ? text then v
-    else literalExpression (lib.generators.toPretty {
-      multiline = true;
-      allowPrettyValues = true;
-    } v);
-
-
   /* For use in the `defaultText` and `example` option attributes. Causes the
      given string to be rendered verbatim in the documentation as Nix code. This
      is necessary for complex values, e.g. functions, or values that depend on

lib/tests/misc.nix

@@ -948,7 +948,7 @@ runTests {
           modules = [ module ];
         }).options;
 
-        locs = filter (o: ! o.internal) (optionAttrSetToDocList options);
+        locs = filter (o: ! o.internal) (optionsToDocTemplate { inherit options; });
       in map (o: o.loc) locs;
     expected = [ [ "_module" "args" ] [ "foo" ] [ "foo" "<name>" "bar" ] [ "foo" "bar" ] ];
   };

nixos/doc/manual/default.nix

@@ -1,41 +1,24 @@
 { pkgs
 , options
 , config
-, version
-, revision
-, extraSources ? []
 , baseOptionsJSON ? null
 , warningsAreErrors ? true
 , allowDocBook ? true
-, prefix ? ../../..
 }:
 
 with pkgs;
 
 let
-  inherit (lib) hasPrefix removePrefix;
-
   lib = pkgs.lib;
 
+  version = lib.trivial.release;
+
   docbook_xsl_ns = pkgs.docbook-xsl-ns.override {
     withManOptDedupPatch = true;
   };
 
-  # We need to strip references to /nix/store/* from options,
-  # including any `extraSources` if some modules came from elsewhere,
-  # or else the build will fail.
-  #
-  # E.g. if some `options` came from modules in ${pkgs.customModules}/nix,
-  # you'd need to include `extraSources = [ pkgs.customModules ]`
-  prefixesToStrip = map (p: "${toString p}/") ([ prefix ] ++ extraSources);
-  stripAnyPrefixes = lib.flip (lib.foldr lib.removePrefix) prefixesToStrip;
-
   optionsDoc = buildPackages.nixosOptionsDoc {
-    inherit options revision baseOptionsJSON warningsAreErrors allowDocBook;
-    transformOptions = opt: opt // {
-      # Clean up declaration sites to not refer to the NixOS source tree.
-      declarations = map stripAnyPrefixes opt.declarations;
-    };
+    inherit options baseOptionsJSON warningsAreErrors allowDocBook;
   };
 
   nixos-lib = import ../../lib { };
@@ -48,19 +31,6 @@ let
       };
     in buildPackages.nixosOptionsDoc {
       inherit (eval) options;
-      inherit (revision);
-      transformOptions = opt: opt // {
-        # Clean up declaration sites to not refer to the NixOS source tree.
-        declarations =
-          map
-            (decl:
-              if hasPrefix (toString ../../..) (toString decl)
-              then
-                let subpath = removePrefix "/" (removePrefix (toString ../../..) (toString decl));
-                in { url = "https://github.com/NixOS/nixpkgs/blob/master/${subpath}"; name = subpath; }
-              else decl)
-            opt.declarations;
-      };
       documentType = "none";
       variablelistId = "test-options-list";
       optionIdPrefix = "test-opt-";

nixos/lib/eval-cacheable-options.nix

@@ -1,23 +1,18 @@
-{ libPath
-, pkgsLibPath
-, nixosPath
-, modules
+{ modules
 , stateVersion
-, release
 }:
 
 let
-  lib = import libPath;
-  modulesPath = "${nixosPath}/modules";
+  lib = import <nixpkgs/lib>;
   # dummy pkgs set that contains no packages, only `pkgs.lib` from the full set.
   # not having `pkgs.lib` causes all users of `pkgs.formats` to fail.
-  pkgs = import pkgsLibPath {
+  pkgs = import <nixpkgs/pkgs/pkgs-lib> {
     inherit lib;
-    pkgs = null;
+    pkgs = throw "pkgs-lib tried to use pkgs for cacheable options build";
   };
-  utils = import "${nixosPath}/lib/utils.nix" {
+  utils = import <nixpkgs/nixos/lib/utils.nix> {
     inherit config lib;
-    pkgs = null;
+    pkgs = throw "utils.nix tried to use pkgs for cacheable options build";
   };
   # this is used both as a module and as specialArgs.
   # as a module it sets the _module special values, as specialArgs it makes `config`
@@ -28,26 +23,17 @@ let
     system.stateVersion = stateVersion;
   };
   eval = lib.evalModules {
-    modules = (map (m: "${modulesPath}/${m}") modules) ++ [
+    modules = (map (m: <nixpkgs/nixos/modules> + "/${m}") modules) ++ [
       config
     ];
     specialArgs = {
       inherit config pkgs utils;
     };
   };
-  docs = import "${nixosPath}/doc/manual" {
-    pkgs = pkgs // {
-      inherit lib;
-      # duplicate of the declaration in all-packages.nix
-      buildPackages.nixosOptionsDoc = attrs:
-        (import "${nixosPath}/lib/make-options-doc")
-          ({ inherit pkgs lib; } // attrs);
-    };
-    config = config.config;
-    options = eval.options;
-    version = release;
-    revision = "release-${release}";
-    prefix = modulesPath;
+  docs = import <nixpkgs/nixos/lib/make-options-doc> {
+    inherit lib;
+    pkgs = throw "make-options-doc tried to use pkgs for cacheable options build";
+    inherit (eval) options;
   };
 in
   docs.optionsNix