From 0407b7a7b4758362e2778138afab4e0f1f22ac53 Mon Sep 17 00:00:00 2001 From: Isaac Mann Date: Mon, 18 Nov 2024 10:02:54 -0500 Subject: [PATCH] docs(core): database cache concept page (#28935) Adds a concept page describing the database cache --------- Co-authored-by: FrozenPandaz --- .../2024-09-25-introducing-nx-powerpack.md | 2 +- docs/generated/manifests/menus.json | 12 ++++---- docs/generated/manifests/nx.json | 18 +++++------ docs/map.json | 6 ++-- docs/shared/deprecated/custom-task-runners.md | 12 -------- docs/shared/deprecated/legacy-cache.md | 30 +++++++++++++++++++ docs/shared/guides/unknown-local-cache.md | 7 ++--- docs/shared/reference/sitemap.md | 2 +- nx-dev/nx-dev/redirect-rules.js | 1 + 9 files changed, 54 insertions(+), 36 deletions(-) delete mode 100644 docs/shared/deprecated/custom-task-runners.md create mode 100644 docs/shared/deprecated/legacy-cache.md diff --git a/docs/blog/2024-09-25-introducing-nx-powerpack.md b/docs/blog/2024-09-25-introducing-nx-powerpack.md index c4902ddaa3625..7934316c0bd29 100644 --- a/docs/blog/2024-09-25-introducing-nx-powerpack.md +++ b/docs/blog/2024-09-25-introducing-nx-powerpack.md @@ -125,7 +125,7 @@ Read all about how to [configure Codeowners for your project in our docs](/nx-en A continuous effort on our Nx core is to improve speed. Last year, we began **rewriting performance-critical parts of Nx into Rust**, and more core components are being rewritten. As part of this effort, we also changed how we manage local cache, moving from a **file-based to a database-based approach**. In addition to small performance gains from reduced I/O, this opens up many opportunities for improving local cache handling, such as keeping only relevant cache based on usage, more easily controlling maximum cache size, and optimizing task orchestration by running failed tasks earlier. -As part of this new approach we're also going to [deprecate custom task runners](/deprecated/custom-task-runners) in Nx 20. I bring this up because it might affect users that relied on 3rd party tools that hooked into the task runners API. +As part of this new approach we're also going to [deprecate custom task runners](/deprecated/legacy-cache) in Nx 20. I bring this up because it might affect users that relied on 3rd party tools that hooked into the task runners API. To fill in on the custom task runner API we're providing a new Powerpack plugin that allows you to use S3 or a network drive as your storing mechanism for your Nx cache. diff --git a/docs/generated/manifests/menus.json b/docs/generated/manifests/menus.json index 9e77cf3d07946..df37cdf4f03af 100644 --- a/docs/generated/manifests/menus.json +++ b/docs/generated/manifests/menus.json @@ -4796,9 +4796,9 @@ "disableCollapsible": false }, { - "name": "Custom Task Runners", - "path": "/deprecated/custom-task-runners", - "id": "custom-task-runners", + "name": "Legacy Cache", + "path": "/deprecated/legacy-cache", + "id": "legacy-cache", "isExternal": false, "children": [], "disableCollapsible": false @@ -4919,9 +4919,9 @@ "disableCollapsible": false }, { - "name": "Custom Task Runners", - "path": "/deprecated/custom-task-runners", - "id": "custom-task-runners", + "name": "Legacy Cache", + "path": "/deprecated/legacy-cache", + "id": "legacy-cache", "isExternal": false, "children": [], "disableCollapsible": false diff --git a/docs/generated/manifests/nx.json b/docs/generated/manifests/nx.json index 2701feb7d6d72..37c8534660328 100644 --- a/docs/generated/manifests/nx.json +++ b/docs/generated/manifests/nx.json @@ -6572,14 +6572,14 @@ "tags": [] }, { - "id": "custom-task-runners", - "name": "Custom Task Runners", + "id": "legacy-cache", + "name": "Legacy Cache", "description": "", "mediaImage": "", - "file": "shared/deprecated/custom-task-runners", + "file": "shared/deprecated/legacy-cache", "itemList": [], "isExternal": false, - "path": "/deprecated/custom-task-runners", + "path": "/deprecated/legacy-cache", "tags": [] }, { @@ -6741,15 +6741,15 @@ "path": "/deprecated/workspace-generators", "tags": [] }, - "/deprecated/custom-task-runners": { - "id": "custom-task-runners", - "name": "Custom Task Runners", + "/deprecated/legacy-cache": { + "id": "legacy-cache", + "name": "Legacy Cache", "description": "", "mediaImage": "", - "file": "shared/deprecated/custom-task-runners", + "file": "shared/deprecated/legacy-cache", "itemList": [], "isExternal": false, - "path": "/deprecated/custom-task-runners", + "path": "/deprecated/legacy-cache", "tags": [] }, "/deprecated/workspace-executors": { diff --git a/docs/map.json b/docs/map.json index fadb275423b79..a4beba3fd5299 100644 --- a/docs/map.json +++ b/docs/map.json @@ -1388,9 +1388,9 @@ "file": "shared/deprecated/workspace-generators" }, { - "name": "Custom Task Runners", - "id": "custom-task-runners", - "file": "shared/deprecated/custom-task-runners" + "name": "Legacy Cache", + "id": "legacy-cache", + "file": "shared/deprecated/legacy-cache" }, { "name": "Workspace Executors", diff --git a/docs/shared/deprecated/custom-task-runners.md b/docs/shared/deprecated/custom-task-runners.md deleted file mode 100644 index 0a2f689e14c19..0000000000000 --- a/docs/shared/deprecated/custom-task-runners.md +++ /dev/null @@ -1,12 +0,0 @@ -# tasksRunnerOptions - -As of Nx 20, the `tasksRunnerOptions` property in `nx.json` is deprecated. This property was used to register custom task runners. `tasksRunnerOptions` and custom task runners will cease to function in Nx 21. In Nx 20, the local cache metadata and project graph are stored in a database, rather than using the file system. (Cache artifacts are still stored on the file system.) This has two benefits: - -1. Cache reads and writes are faster. -2. The local cache is more secure since other processes with access to the file system can no longer read or modify the cache. - -For most organizations, this feature is a net positive. If you are currently using a custom task runner, you are most likely using it to define your own custom [remote cache](/ci/features/remote-cache) storage location. You have several options moving forward: - -1. Use [Nx Cloud](/nx-cloud) for your remote cache -2. Use an [Nx Powerpack](/powerpack) plugin to store your remote cache on [Amazon S3](/nx-api/powerpack-s3-cache), [Google Cloud](/nx-api/powerpack-gcs-cache), [Google Cloud](/nx-api/powerpack-gcs-cache) or a [network drive](/nx-api/powerpack-shared-fs-cache) -3. Use the deprecated custom task runner feature until Nx 21 diff --git a/docs/shared/deprecated/legacy-cache.md b/docs/shared/deprecated/legacy-cache.md new file mode 100644 index 0000000000000..5655fb4f57f6b --- /dev/null +++ b/docs/shared/deprecated/legacy-cache.md @@ -0,0 +1,30 @@ +# Legacy Cache + +In Nx 21, the legacy file system cache will be removed in favor of a new database cache. The new database cache stores metadata in a database, rather than blindly trusting the file system. The database cache has the following benefits: + +1. Cache reads and writes are faster. +2. The local cache is more secure since Nx will no longer retrieve artifacts it does not recognize. + +The legacy file system cache can still be used in Nx 20 by setting `useLegacyCache: true` in your `nx.json` file. To gain the benefits above, remove `useLegacyCache: true` from your `nx.json`. + +If you are currently using a custom task runner or the `NX_REJECT_UNKNOWN_LOCAL_CACHE` environment variable continue reading below. + +## tasksRunnerOptions + +As of Nx 20, the `tasksRunnerOptions` property in `nx.json` is deprecated. This property was used to register custom task runners. `tasksRunnerOptions` and custom task runners will not work with the new database cache. + +If you are using `tasksRunnerOptions`, you have a few options moving forward: + +1. Use [Nx Cloud](/nx-cloud) for your remote cache. This is the safest, lowest-maintenance, most recommended option +2. If you cannot use Nx Cloud, consider an [Nx Powerpack](/powerpack) plugin for caching: [Amazon S3](/nx-api/powerpack-s3-cache), [Google Cloud](/nx-api/powerpack-gcs-cache), [Azure](/nx-api/powerpack-azure-cache) or a [shared network drive](/nx-api/powerpack-shared-fs-cache) +3. If there is no Powerpack plugin that supports the service where you want to store the remote cache, [file an issue](https://github.com/nrwl/nx/issues/new) + +## NX_REJECT_UNKNOWN_LOCAL_CACHE + +The `NX_REJECT_UNKNOWN_LOCAL_CACHE` environment variable does not work with the new database cache. We have introduced a new cache which will recognize artifacts from other machines. + +If you are using `NX_REJECT_UNKNOWN_LOCAL_CACHE` to share your local cache on a network drive, you have a few options moving forward: + +1. Use [Nx Cloud](/nx-cloud) for your remote cache. This is the safest, lowest-maintenance, most recommended option +2. Use the [Nx Powerpack Shared Fs Cache](/nx-api/powerpack-shared-fs-cache) which recognizes artifacts from other machines +3. Use another [Nx Powerpack](/powerpack) plugin for caching on a cloud provider: [Amazon S3](/nx-api/powerpack-s3-cache), [Google Cloud](/nx-api/powerpack-gcs-cache), or [Azure](/nx-api/powerpack-azure-cache) diff --git a/docs/shared/guides/unknown-local-cache.md b/docs/shared/guides/unknown-local-cache.md index 9068ed19f83b6..d7d7ca715cfca 100644 --- a/docs/shared/guides/unknown-local-cache.md +++ b/docs/shared/guides/unknown-local-cache.md @@ -44,14 +44,13 @@ error. ## You Share Cache with Another Machine Using a Network Drive -You can prefix any Nx command with `NX_REJECT_UNKNOWN_LOCAL_CACHE=0` to ignore the errors ( +When using the legacy file system cache ([deprecated in Nx 20](/deprecated/legacy-cache)), you can prefix any Nx command with `NX_REJECT_UNKNOWN_LOCAL_CACHE=0` to ignore the errors ( e.g., `NX_REJECT_UNKNOWN_LOCAL_CACHE=0 nx run-many -t build test`). This is similar to setting `NODE_TLS_REJECT_UNAUTHORIZED=0` to ignore any errors stemming form self-signed certificates. Even though it will make it work, this approach is discouraged. -Storing Nx's local cache on a network drive can present security risks. When a network drive is shared, every CI run has -access to all the previously created Nx cache artifacts. Hence, it is plausible for every single artifact - for every -single task hash - to be accessed without leaving any trace. This is feasible due to the network drive's capability to +Storing Nx's local cache on a network drive presents security risks, so we prevent you from doing so. When a network drive is shared, every CI run has access to all the previously created Nx cache artifacts. Hence, it is plausible for every single artifact - for every +single task hash - to be accessed without leaving any trace. This is possible due to the network drive's capability to allow overwrites. ## How Nx Replay Makes Sure Sharing Cache is Safe diff --git a/docs/shared/reference/sitemap.md b/docs/shared/reference/sitemap.md index c010ed03f3d29..48b8c9aa5823e 100644 --- a/docs/shared/reference/sitemap.md +++ b/docs/shared/reference/sitemap.md @@ -221,7 +221,7 @@ - [workspace.json](/deprecated/workspace-json) - [As Provided vs. Derived](/deprecated/as-provided-vs-derived) - [Workspace Generators](/deprecated/workspace-generators) - - [Custom Task Runners](/deprecated/custom-task-runners) + - [Legacy Cache](/deprecated/legacy-cache) - [Workspace Executors](/deprecated/workspace-executors) - [runtimeCacheInputs](/deprecated/runtime-cache-inputs) - [cacheableOperations](/deprecated/cacheable-operations) diff --git a/nx-dev/nx-dev/redirect-rules.js b/nx-dev/nx-dev/redirect-rules.js index 944499744e517..772120373fabd 100644 --- a/nx-dev/nx-dev/redirect-rules.js +++ b/nx-dev/nx-dev/redirect-rules.js @@ -1099,6 +1099,7 @@ const removedDeprecatedUrls = { '/recipes/storybook/overview-react', // 417 views: mostly people searching "React Storybook" is matching this outdated page that was for Nx 12.7 '/deprecated/storybook/upgrade-storybook-v6-react': '/recipes/storybook/overview-react', // 80 views + '/deprecated/custom-task-runners': '/deprecated/legacy-cache', }; const decisionsSection = {