From c06703873917034f550ba5dbdd3aa9cc5017e8e6 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Mon, 30 May 2022 13:40:01 +0200 Subject: [PATCH] Docs for snapshots as simple archives (#87211) Backport of #86261 --- .../cluster-index-compat.asciidoc | 19 ++-- docs/reference/upgrade.asciidoc | 17 ++-- .../upgrade/archive-indices.asciidoc | 88 +++++++++++++++++++ 3 files changed, 109 insertions(+), 15 deletions(-) create mode 100644 docs/reference/upgrade/archive-indices.asciidoc diff --git a/docs/reference/snapshot-restore/cluster-index-compat.asciidoc b/docs/reference/snapshot-restore/cluster-index-compat.asciidoc index 24e348105ee46..3fdc5c38e51df 100644 --- a/docs/reference/snapshot-restore/cluster-index-compat.asciidoc +++ b/docs/reference/snapshot-restore/cluster-index-compat.asciidoc @@ -1,12 +1,13 @@ -[cols="^,^,^,^,^"] +[cols="^,^,^,^,^,^"] |==== -| 4+^h| Cluster version -h| Index creation version | 6.8 | 7.0–7.1 | 7.2–{prev-major-last} | 8.0–{minor-version} -| 5.0–5.6 | {yes-icon} | {no-icon} | {no-icon} | {no-icon} -| 6.0–6.7 | {yes-icon} | {yes-icon} | {yes-icon} | {no-icon} -| 6.8 | {yes-icon} | {no-icon} | {yes-icon} | {no-icon} -| 7.0–7.1 | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} -| 7.2–{prev-major-last} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} -| 8.0–{minor-version} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} +| 5+^h| Cluster version +h| Index creation version | 6.8 | 7.0–7.1 | 7.2–{prev-major-last} | 8.0–8.2 | 8.3-{minor-version} +| 5.0–5.6 | {yes-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon}footnote:archive[Using <> functionality] +| 6.0–6.7 | {yes-icon} | {yes-icon} | {yes-icon} | {no-icon} | {yes-icon}footnote:archive[] +| 6.8 | {yes-icon} | {no-icon} | {yes-icon} | {no-icon} | {yes-icon}footnote:archive[] +| 7.0–7.1 | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 7.2–{prev-major-last} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.0–8.2 | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} +| 8.3–{minor-version} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} |==== diff --git a/docs/reference/upgrade.asciidoc b/docs/reference/upgrade.asciidoc index 677877053147e..5bcdc9a4b7c94 100644 --- a/docs/reference/upgrade.asciidoc +++ b/docs/reference/upgrade.asciidoc @@ -19,12 +19,15 @@ from 7.x]. [[upgrade-index-compatibility]] === Index compatibility -{es} can read indices created in the previous major version. If you -have indices created in 6.x or earlier, you must reindex or delete them -before upgrading to {version}. {es} nodes will fail to start if -incompatible indices are present. Snapshots of 6.x or earlier indices cannot be -restored to a 8.x cluster even if they were created by a 7.x cluster. -The **Upgrade Assistant** in {prev-major-last} identifies any indices +{es} has full query and write support for indices created in the previous major +version. If you have indices created in 6.x or earlier, you might use the +<> to import them into newer {es} +versions, or you must reindex or delete them before upgrading to {version}. +{es} nodes will fail to start if incompatible indices are present. +Snapshots of 6.x or earlier indices can only restored using the +<> to a 8.x cluster even if they +were created by a 7.x cluster. +The **Upgrade Assistant** in {prev-major-last} identifies any indices that need to be reindexed or removed. [discrete] @@ -42,3 +45,5 @@ the REST API. include::{xes-repo-dir}/security/fips-java17.asciidoc[] include::upgrade/archived-settings.asciidoc[] + +include::upgrade/archive-indices.asciidoc[] diff --git a/docs/reference/upgrade/archive-indices.asciidoc b/docs/reference/upgrade/archive-indices.asciidoc new file mode 100644 index 0000000000000..5e905aa9215a4 --- /dev/null +++ b/docs/reference/upgrade/archive-indices.asciidoc @@ -0,0 +1,88 @@ +[[archive-indices]] +== Reading indices from older {es} versions + +{es} has full query and write support for indices created in the previous major +version. If you have indices created in {es} versions 5 or 6, you can now use +the archive functionality to import them into newer {es} versions as well. + +The archive functionality provides slower read-only access to older {es} data, +for compliance or regulatory reasons, the occasional lookback or investigation, +or to rehydrate parts of it. Access to the data is expected to be infrequent, +and can therefore happen with limited performance and query capabilities. + +For this, {es} has the ability to access older snapshot repositories +(going back to version 5). The legacy indices in the <> +can either be <>, or can be directly accessed +via <> so that the archived data +won't even need to fully reside on local disks for access. + +[discrete] +[[archive-indices-supported-field-types]] +=== Supported field types + +Old mappings are imported as much "as-is" as possible into {es} 8, but only +provide regular query capabilities on a select subset of fields: + +- <> +- <> +- <> +- <> +- <>: the date `format` setting on date fields is supported + as long as it behaves similarly across these versions. In case it is not, + for example {ref-7x}/migrate-to-java-time.html[when using custom date formats], + this field can be updated on legacy indices so that it can be changed by a + user if need be. +- <>: the `normalizer` setting on keyword + fields is supported as long as it behaves similarly across these versions. + In case it is not, this field can be updated on legacy indices if need be. +- <>: scoring capabilities are limited, and all + queries return constant scores that are equal to 1.0. The `analyzer` + settings on text fields are supported as long as they behave similarly + across these versions. In case they do not, they can be updated on legacy + indices if need be. +- <> +- <> +- <> fields +- some basic metadata fields, e.g. `_type` for querying {es} 5 indices +- <> +- <> + +{es} 5 indices with mappings that have {ref-7x}/removal-of-types.html[multiple mapping types] +are collapsed together on a best-effort basis before they are imported. + +In case the auto-import of mappings does not work, or the new {es} version +can't make sense of the mapping, it falls back to importing the index without +the mapping, but stores the original mapping in the <> +section of the imported index. The legacy mapping can then be introspected +using the <> API and an updated mapping can +be manually put in place using the <> API, +copying and adapting relevant sections of the legacy mapping to work with the +current {es} version. While auto-import is expected to work in most cases, +failures of doing so should be https://github.com/elastic/elasticsearch/issues/new/choose[raised] +with the Elastic team for future improvements. + +[discrete] +=== Supported APIs + +Archive indices are read-only, and provide data access via the +<> and <> APIs. +They do not support the <> nor any write APIs. + +Archive indices allow running queries as well as aggregations in so far as +they are <>. + +Due to `_source` access the data can also be <> +to a new index that has full compatibility with the current {es} version. + +[discrete] +=== How to upgrade older {es} 5 or 6 clusters? + +Take a snapshot of the indices in the old cluster, delete indices that are not +directly supported by ES 8 (i.e. indices older than 7.0), upgrade the cluster +without the old indices, and then <> the legacy +indices from the snapshot or <> +them via searchable snapshots. + +In the future, we plan on streamlining the upgrade process going forward, +making it easier to take legacy indices along when going to future major +{es} versions.