From 7c5e178e529a030d2da1c837d13632ff9a5c777e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 9 Jan 2024 17:54:58 -0500 Subject: [PATCH] WIP systematize `builtins.fetchTree` docs The renaming task is splatting everything together into markdown lists. I think I want to do that with the lowdown AST; I opened https://github.com/kristapsdz/lowdown/issues/131 for this purpose PR #9273 should be able to improve upon this a good bit, but I think it is still a useful stepping stone. --- src/libexpr/primops/fetchTree.cc | 125 ------------------------ src/libfetchers/fetchers.hh | 11 ++- src/libfetchers/git.cc | 160 +++++++++++++++++++++++++++---- src/libfetchers/github.cc | 62 ++++++++++-- src/libfetchers/indirect.cc | 32 ++++++- src/libfetchers/mercurial.cc | 44 +++++++-- src/libfetchers/path.cc | 38 ++++++-- src/libfetchers/tarball.cc | 108 +++++++++++++++++++-- 8 files changed, 400 insertions(+), 180 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index eb2df8626c9..05cd8b3db73 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -227,131 +227,6 @@ static RegisterPrimOp primop_fetchTree({ document) if `fetchTree` was a curried call with the first paramter for `type` or an attribute like `builtins.fetchTree.git`! --> - - `"file"` - - Place a plain file into the Nix store. - This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl) - - - `url` (String, required) - - Supported protocols: - - - `https` - - > **Example** - > - > ```nix - > fetchTree { - > type = "file"; - > url = "https://example.com/index.html"; - > } - > ``` - - - `http` - - Insecure HTTP transfer for legacy sources. - - > **Warning** - > - > HTTP performs no encryption or authentication. - > Use a `narHash` known in advance to ensure the output has expected contents. - - - `file` - - A file on the local file system. - - > **Example** - > - > ```nix - > fetchTree { - > type = "file"; - > url = "file:///home/eelco/nix/README.md"; - > } - > ``` - - - `"tarball"` - - Download a tar archive and extract it into the Nix store. - This has the same underyling implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball) - - - `url` (String, required) - - > **Example** - > - > ```nix - > fetchTree { - > type = "tarball"; - > url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11"; - > } - > ``` - - - `"git"` - - Fetch a Git tree and copy it to the Nix store. - This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit). - - - `url` (String, required) - - The URL formats supported are the same as for Git itself. - - > **Example** - > - > ```nix - > fetchTree { - > type = "git"; - > url = "git@github.com:NixOS/nixpkgs.git"; - > } - > ``` - - > **Note** - > - > If the URL points to a local directory, and no `ref` or `rev` is given, Nix will only consider files added to the Git index, as listed by `git ls-files` but use the *current file contents* of the Git working directory. - - - `ref` (String, optional) - - A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name. - - Default: `"HEAD"` - - - `rev` (String, optional) - - A Git revision; a commit hash. - - Default: the tip of `ref` - - - `shallow` (Bool, optional) - - Make a shallow clone when fetching the Git tree. - - Default: `false` - - - `submodules` (Bool, optional) - - Also fetch submodules if available. - - Default: `false` - - - `allRefs` (Bool, optional) - - If set to `true`, always fetch the entire repository, even if the latest commit is still in the cache. - Otherwise, only the latest commit is fetched if it is not already cached. - - Default: `false` - - - `lastModified` (Integer, optional) - - Unix timestamp of the fetched commit. - - If set, pass through the value to the output attribute set. - Otherwise, generated from the fetched Git tree. - - - `revCount` (Integer, optional) - - Number of revisions in the history of the Git repository before the fetched commit. - - If set, pass through the value to the output attribute set. - Otherwise, generated from the fetched Git tree. - The following input types are still subject to change: - `"path"` diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 5f3254b6d88..b8e38d651dc 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -149,14 +149,19 @@ struct InputScheme */ virtual std::string_view schemeName() const = 0; + /** + * Longform description of this scheme, for documentation purposes. + */ + virtual std::string schemeDescription() const = 0; + /** * Allowed attributes in an attribute set that is converted to an - * input. + * input, and documentation for each attribute. * - * `type` is not included from this set, because the `type` field is + * `type` is not included from this map, because the `type` field is parsed first to choose which scheme; `type` is always required. */ - virtual StringSet allowedAttrs() const = 0; + virtual StringMap allowedAttrs() const = 0; virtual ParsedURL toURL(const Input & input) const; diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 79270c3179f..e80c98aa2a4 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -191,25 +191,151 @@ struct GitInputScheme : InputScheme return "git"; } - StringSet allowedAttrs() const override + std::string schemeDescription() const override + { + return stripIndentation(R"( + Fetch a Git tree and copy it to the Nix store. + This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit). + )"); + } + + StringMap allowedAttrs() const override { return { - "url", - "ref", - "rev", - "shallow", - "submodules", - "lastModified", - "revCount", - "narHash", - "allRefs", - "name", - "dirtyRev", - "dirtyShortRev", - "verifyCommit", - "keytype", - "publicKey", - "publicKeys", + { + "url", + R"( + (String, required) + + The URL formats supported are the same as for Git itself. + + > **Example** + > + > ```nix + > fetchTree { + > type = "git"; + > url = "git@github.com:NixOS/nixpkgs.git"; + > } + > ``` + + > **Note** + > + > If the URL points to a local directory, and no `ref` or `rev` is given, Nix will only consider files added to the Git index, as listed by `git ls-files` but use the *current file contents* of the Git working directory. + )", + }, + { + "ref", + R"( + (String, optional) + + A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name. + + Default: `"HEAD"` + )", + }, + { + "rev", + R"( + (String, optional) + + A Git revision; a commit hash. + + Default: the tip of `ref` + )", + }, + { + "shallow", + R"( + (Bool, optional) + + Make a shallow clone when fetching the Git tree. + + Default: `false` + )", + }, + { + "submodules", + R"( + (Bool, optional) + + Also fetch submodules if available. + + Default: `false` + )", + }, + { + "lastModified", + R"( + (Integer, optional) + + Unix timestamp of the fetched commit. + + If set, pass through the value to the output attribute set. + Otherwise, generated from the fetched Git tree. + )", + }, + { + "revCount", + R"( + (Integer, optional) + + Number of revisions in the history of the Git repository before the fetched commit. + + If set, pass through the value to the output attribute set. + Otherwise, generated from the fetched Git tree. + )", + }, + { + "narHash", + R"( + )", + }, + { + "allRefs", + R"( + (Bool, optional) + + If set to `true`, always fetch the entire repository, even if the latest commit is still in the cache. + Otherwise, only the latest commit is fetched if it is not already cached. + + Default: `false` + )", + }, + { + "name", + R"( + )", + }, + { + "dirtyRev", + R"( + )", + }, + { + "dirtyShortRev", + R"( + )", + }, + { + "verifyCommit", + R"( + )", + }, + { + "keytype", + R"( + )", + }, + { + "publicKey", + R"( + )", + }, + { + "publicKeys", + R"( + )", + }, }; } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 498e4135793..57572a8f370 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -99,16 +99,44 @@ struct GitArchiveInputScheme : InputScheme return input; } - StringSet allowedAttrs() const override + StringMap allowedAttrs() const override { return { - "owner", - "repo", - "ref", - "rev", - "narHash", - "lastModified", - "host", + { + "owner", + R"( + )", + }, + { + "repo", + R"( + )", + }, + { + "ref", + R"( + )", + }, + { + "rev", + R"( + )", + }, + { + "narHash", + R"( + )", + }, + { + "lastModified", + R"( + )", + }, + { + "host", + R"( + )", + }, }; } @@ -243,6 +271,12 @@ struct GitHubInputScheme : GitArchiveInputScheme { std::string_view schemeName() const override { return "github"; } + std::string schemeDescription() const override + { + // TODO + return ""; + } + std::optional> accessHeaderFromToken(const std::string & token) const override { // Github supports PAT/OAuth2 tokens and HTTP Basic @@ -324,6 +358,12 @@ struct GitLabInputScheme : GitArchiveInputScheme { std::string_view schemeName() const override { return "gitlab"; } + std::string schemeDescription() const override + { + // TODO + return ""; + } + std::optional> accessHeaderFromToken(const std::string & token) const override { // Gitlab supports 4 kinds of authorization, two of which are @@ -392,6 +432,12 @@ struct SourceHutInputScheme : GitArchiveInputScheme { std::string_view schemeName() const override { return "sourcehut"; } + std::string schemeDescription() const override + { + // TODO + return ""; + } + std::optional> accessHeaderFromToken(const std::string & token) const override { // SourceHut supports both PAT and OAuth2. See diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 002c0c292f4..2f58b218ac2 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -55,13 +55,35 @@ struct IndirectInputScheme : InputScheme return "indirect"; } - StringSet allowedAttrs() const override + std::string schemeDescription() const override + { + // TODO + return ""; + } + + StringMap allowedAttrs() const override { return { - "id", - "ref", - "rev", - "narHash", + { + "id", + R"( + )", + }, + { + "ref", + R"( + )", + }, + { + "rev", + R"( + )", + }, + { + "narHash", + R"( + )", + }, }; } diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 9982389ab20..93d8a274338 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -77,15 +77,45 @@ struct MercurialInputScheme : InputScheme return "hg"; } - StringSet allowedAttrs() const override + std::string schemeDescription() const override + { + // TODO + return ""; + } + + StringMap allowedAttrs() const override { return { - "url", - "ref", - "rev", - "revCount", - "narHash", - "name", + { + "url", + R"( + )", + }, + { + "ref", + R"( + )", + }, + { + "rev", + R"( + )", + }, + { + "revCount", + R"( + )", + }, + { + "narHash", + R"( + )", + }, + { + "name", + R"( + )", + }, }; } diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index f9b973320b0..a276cd1ccd6 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -37,19 +37,45 @@ struct PathInputScheme : InputScheme return "path"; } - StringSet allowedAttrs() const override + std::string schemeDescription() const override + { + // TODO + return ""; + } + + StringMap allowedAttrs() const override { return { - "path", + { + "path", + R"( + )", + }, /* Allow the user to pass in "fake" tree info attributes. This is useful for making a pinned tree work the same as the repository from which is exported (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */ - "rev", - "revCount", - "lastModified", - "narHash", + { + "rev", + R"( + )", + }, + { + "revCount", + R"( + )", + }, + { + "lastModified", + R"( + )", + }, + { + "narHash", + R"( + )", + }, }; } std::optional inputFromAttrs(const Attrs & attrs) const override diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 3b7709440b2..5520413e037 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -228,17 +228,90 @@ struct CurlInputScheme : InputScheme return input; } - StringSet allowedAttrs() const override + StringMap allowedAttrs() const override { return { - "type", - "url", - "narHash", - "name", - "unpack", - "rev", - "revCount", - "lastModified", + { + "url", + R"( + (String, required) + + Supported protocols: + + - `https` + + > **Example** + > + > ```nix + > fetchTree { + > type = "file"; + > url = "https://example.com/index.html"; + > } + > ``` + + - `http` + + Insecure HTTP transfer for legacy sources. + + > **Warning** + > + > HTTP performs no encryption or authentication. + > Use a `narHash` known in advance to ensure the output has expected cont + ents. + + - `file` + + A file on the local file system. + + > **Example** + > + > ```nix + > fetchTree { + > type = "file"; + > url = "file:///home/eelco/nix/README.md"; + > } + > ``` + + > **Example** + > + > ```nix + > fetchTree { + > type = "tarball"; + > url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11"; + > } + > ``` + )", + }, + { + "narHash", + R"( + )", + }, + { + "name", + R"( + )", + }, + { + "unpack", + R"( + )", + }, + { + "rev", + R"( + )", + }, + { + "revCount", + R"( + )", + }, + { + "lastModified", + R"( + )", + }, }; } @@ -266,6 +339,14 @@ struct FileInputScheme : CurlInputScheme { std::string_view schemeName() const override { return "file"; } + std::string schemeDescription() const override + { + return stripIndentation(R"( + Place a plain file into the Nix store. + This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl) + )"); + } + bool isValidURL(const ParsedURL & url, bool requireTree) const override { auto parsedUrlScheme = parseUrlScheme(url.scheme); @@ -286,6 +367,15 @@ struct TarballInputScheme : CurlInputScheme { std::string_view schemeName() const override { return "tarball"; } + std::string schemeDescription() const override + { + return stripIndentation(R"( + Download a tar archive and extract it into the Nix store. + This has the same underyling implementation as [`builtins.fetchTarball`](@doc + root@/language/builtins.md#builtins-fetchTarball) + )"); + } + bool isValidURL(const ParsedURL & url, bool requireTree) const override { auto parsedUrlScheme = parseUrlScheme(url.scheme);