[SO migration] remove v1 implementation (#118000)

* remove v1 implementation * fix type * remove unused mock * expose kibanaVersion again * fix migrator mock * move KibanaMigrator out of the kibana subfolder * fix imports * moves migrationsv2 into migrations * fix test mocking
elastic · Nov 10, 2021 · f716387 · f716387
1 parent 7ab3593
commit f716387
Show file tree

Hide file tree

Showing 147 changed files with 717 additions and 3,994 deletions.
diff --git a/src/core/server/saved_objects/migrations/README.md b/src/core/server/saved_objects/migrations/README.md
diff --git a/..._snapshots__/kibana_migrator.test.ts.snap → ..._snapshots__/kibana_migrator.test.ts.snap b/..._snapshots__/kibana_migrator.test.ts.snap → ..._snapshots__/kibana_migrator.test.ts.snap
diff --git a/...rations_state_action_machine.test.ts.snap → ...rations_state_action_machine.test.ts.snap b/...rations_state_action_machine.test.ts.snap → ...rations_state_action_machine.test.ts.snap
diff --git a/...k_overwrite_transformed_documents.test.ts → ...k_overwrite_transformed_documents.test.ts b/...k_overwrite_transformed_documents.test.ts → ...k_overwrite_transformed_documents.test.ts
diff --git a/...s/bulk_overwrite_transformed_documents.ts → ...s/bulk_overwrite_transformed_documents.ts b/...s/bulk_overwrite_transformed_documents.ts → ...s/bulk_overwrite_transformed_documents.ts
diff --git a/...actions/calculate_exclude_filters.test.ts → ...actions/calculate_exclude_filters.test.ts b/...actions/calculate_exclude_filters.test.ts → ...actions/calculate_exclude_filters.test.ts
diff --git a/...nsv2/actions/calculate_exclude_filters.ts → ...ions/actions/calculate_exclude_filters.ts b/...nsv2/actions/calculate_exclude_filters.ts → ...ions/actions/calculate_exclude_filters.ts
diff --git a/.../catch_retryable_es_client_errors.test.ts → .../catch_retryable_es_client_errors.test.ts b/.../catch_retryable_es_client_errors.test.ts → .../catch_retryable_es_client_errors.test.ts
diff --git a/...tions/catch_retryable_es_client_errors.ts → ...tions/catch_retryable_es_client_errors.ts b/...tions/catch_retryable_es_client_errors.ts → ...tions/catch_retryable_es_client_errors.ts
diff --git a/...v2/actions/check_for_unknown_docs.test.ts → ...ns/actions/check_for_unknown_docs.test.ts b/...v2/actions/check_for_unknown_docs.test.ts → ...ns/actions/check_for_unknown_docs.test.ts
diff --git a/...tionsv2/actions/check_for_unknown_docs.ts → ...rations/actions/check_for_unknown_docs.ts b/...tionsv2/actions/check_for_unknown_docs.ts → ...rations/actions/check_for_unknown_docs.ts
diff --git a/.../migrationsv2/actions/clone_index.test.ts → ...ts/migrations/actions/clone_index.test.ts b/.../migrationsv2/actions/clone_index.test.ts → ...ts/migrations/actions/clone_index.test.ts
diff --git a/...jects/migrationsv2/actions/clone_index.ts → ...objects/migrations/actions/clone_index.ts b/...jects/migrationsv2/actions/clone_index.ts → ...objects/migrations/actions/clone_index.ts
diff --git a/...ts/migrationsv2/actions/close_pit.test.ts → ...ects/migrations/actions/close_pit.test.ts b/...ts/migrationsv2/actions/close_pit.test.ts → ...ects/migrations/actions/close_pit.test.ts
diff --git a/...objects/migrationsv2/actions/close_pit.ts → ...d_objects/migrations/actions/close_pit.ts b/...objects/migrationsv2/actions/close_pit.ts → ...d_objects/migrations/actions/close_pit.ts
diff --git a/...objects/migrationsv2/actions/constants.ts → ...d_objects/migrations/actions/constants.ts b/...objects/migrationsv2/actions/constants.ts → ...d_objects/migrations/actions/constants.ts
diff --git a/...migrationsv2/actions/create_index.test.ts → ...s/migrations/actions/create_index.test.ts b/...migrationsv2/actions/create_index.test.ts → ...s/migrations/actions/create_index.test.ts
diff --git a/...ects/migrationsv2/actions/create_index.ts → ...bjects/migrations/actions/create_index.ts b/...ects/migrationsv2/actions/create_index.ts → ...bjects/migrations/actions/create_index.ts
diff --git a/...ts/migrationsv2/actions/es_errors.test.ts → ...ects/migrations/actions/es_errors.test.ts b/...ts/migrationsv2/actions/es_errors.test.ts → ...ects/migrations/actions/es_errors.test.ts
diff --git a/...objects/migrationsv2/actions/es_errors.ts → ...d_objects/migrations/actions/es_errors.ts b/...objects/migrationsv2/actions/es_errors.ts → ...d_objects/migrations/actions/es_errors.ts
diff --git a/...igrationsv2/actions/fetch_indices.test.ts → .../migrations/actions/fetch_indices.test.ts b/...igrationsv2/actions/fetch_indices.test.ts → .../migrations/actions/fetch_indices.test.ts
diff --git a/...cts/migrationsv2/actions/fetch_indices.ts → ...jects/migrations/actions/fetch_indices.ts b/...cts/migrationsv2/actions/fetch_indices.ts → ...jects/migrations/actions/fetch_indices.ts
diff --git a/...ved_objects/migrationsv2/actions/index.ts → ...saved_objects/migrations/actions/index.ts b/...ved_objects/migrationsv2/actions/index.ts → ...saved_objects/migrations/actions/index.ts
diff --git a/...actions/integration_tests/actions.test.ts → ...actions/integration_tests/actions.test.ts b/...actions/integration_tests/actions.test.ts → ...actions/integration_tests/actions.test.ts
@@ -39,7 +39,7 @@ import {
 import * as Either from 'fp-ts/lib/Either';
 import * as Option from 'fp-ts/lib/Option';
 import { errors } from '@elastic/elasticsearch';
-import { DocumentsTransformFailed, DocumentsTransformSuccess } from '../../../migrations/core';
+import { DocumentsTransformFailed, DocumentsTransformSuccess } from '../../core';
 import { TaskEither } from 'fp-ts/lib/TaskEither';
 import Path from 'path';
 

diff --git a/...n_tests/archives/7.7.2_xpack_100k_obj.zip → ...n_tests/archives/7.7.2_xpack_100k_obj.zip b/...n_tests/archives/7.7.2_xpack_100k_obj.zip → ...n_tests/archives/7.7.2_xpack_100k_obj.zip
diff --git a/...tions/integration_tests/es_errors.test.ts → ...tions/integration_tests/es_errors.test.ts b/...tions/integration_tests/es_errors.test.ts → ...tions/integration_tests/es_errors.test.ts
diff --git a/...cts/migrationsv2/actions/open_pit.test.ts → ...jects/migrations/actions/open_pit.test.ts b/...cts/migrationsv2/actions/open_pit.test.ts → ...jects/migrations/actions/open_pit.test.ts
diff --git a/..._objects/migrationsv2/actions/open_pit.ts → ...ed_objects/migrations/actions/open_pit.ts b/..._objects/migrationsv2/actions/open_pit.ts → ...ed_objects/migrations/actions/open_pit.ts
diff --git a/...2/actions/pickup_updated_mappings.test.ts → ...s/actions/pickup_updated_mappings.test.ts b/...2/actions/pickup_updated_mappings.test.ts → ...s/actions/pickup_updated_mappings.test.ts
diff --git a/...ionsv2/actions/pickup_updated_mappings.ts → ...ations/actions/pickup_updated_mappings.ts b/...ionsv2/actions/pickup_updated_mappings.ts → ...ations/actions/pickup_updated_mappings.ts
diff --git a/...igrationsv2/actions/read_with_pit.test.ts → .../migrations/actions/read_with_pit.test.ts b/...igrationsv2/actions/read_with_pit.test.ts → .../migrations/actions/read_with_pit.test.ts
diff --git a/...cts/migrationsv2/actions/read_with_pit.ts → ...jects/migrations/actions/read_with_pit.ts b/...cts/migrationsv2/actions/read_with_pit.ts → ...jects/migrations/actions/read_with_pit.ts
diff --git a/...igrationsv2/actions/refresh_index.test.ts → .../migrations/actions/refresh_index.test.ts b/...igrationsv2/actions/refresh_index.test.ts → .../migrations/actions/refresh_index.test.ts
diff --git a/...cts/migrationsv2/actions/refresh_index.ts → ...jects/migrations/actions/refresh_index.ts b/...cts/migrationsv2/actions/refresh_index.ts → ...jects/migrations/actions/refresh_index.ts
diff --git a/...ects/migrationsv2/actions/reindex.test.ts → ...bjects/migrations/actions/reindex.test.ts b/...ects/migrationsv2/actions/reindex.test.ts → ...bjects/migrations/actions/reindex.test.ts
diff --git a/...d_objects/migrationsv2/actions/reindex.ts → ...ved_objects/migrations/actions/reindex.ts b/...d_objects/migrationsv2/actions/reindex.ts → ...ved_objects/migrations/actions/reindex.ts
diff --git a/...ionsv2/actions/remove_write_block.test.ts → ...ations/actions/remove_write_block.test.ts b/...ionsv2/actions/remove_write_block.test.ts → ...ations/actions/remove_write_block.test.ts
diff --git a/...igrationsv2/actions/remove_write_block.ts → .../migrations/actions/remove_write_block.ts b/...igrationsv2/actions/remove_write_block.ts → .../migrations/actions/remove_write_block.ts
diff --git a/...ons/search_for_outdated_documents.test.ts → ...ons/search_for_outdated_documents.test.ts b/...ons/search_for_outdated_documents.test.ts → ...ons/search_for_outdated_documents.test.ts
diff --git a/.../actions/search_for_outdated_documents.ts → .../actions/search_for_outdated_documents.ts b/.../actions/search_for_outdated_documents.ts → .../actions/search_for_outdated_documents.ts
diff --git a/...rationsv2/actions/set_write_block.test.ts → ...igrations/actions/set_write_block.test.ts b/...rationsv2/actions/set_write_block.test.ts → ...igrations/actions/set_write_block.test.ts
diff --git a/...s/migrationsv2/actions/set_write_block.ts → ...cts/migrations/actions/set_write_block.ts b/...s/migrationsv2/actions/set_write_block.ts → ...cts/migrations/actions/set_write_block.ts
diff --git a/...ts/migrationsv2/actions/transform_docs.ts → ...ects/migrations/actions/transform_docs.ts b/...ts/migrationsv2/actions/transform_docs.ts → ...ects/migrations/actions/transform_docs.ts
diff --git a/...grationsv2/actions/update_aliases.test.ts → ...migrations/actions/update_aliases.test.ts b/...grationsv2/actions/update_aliases.test.ts → ...migrations/actions/update_aliases.test.ts
diff --git a/...ts/migrationsv2/actions/update_aliases.ts → ...ects/migrations/actions/update_aliases.ts b/...ts/migrationsv2/actions/update_aliases.ts → ...ects/migrations/actions/update_aliases.ts
diff --git a/...ctions/update_and_pickup_mappings.test.ts → ...ctions/update_and_pickup_mappings.test.ts b/...ctions/update_and_pickup_mappings.test.ts → ...ctions/update_and_pickup_mappings.test.ts
diff --git a/...sv2/actions/update_and_pickup_mappings.ts → ...ons/actions/update_and_pickup_mappings.ts b/...sv2/actions/update_and_pickup_mappings.ts → ...ons/actions/update_and_pickup_mappings.ts
diff --git a/...ts/migrationsv2/actions/verify_reindex.ts → ...ects/migrations/actions/verify_reindex.ts b/...ts/migrationsv2/actions/verify_reindex.ts → ...ects/migrations/actions/verify_reindex.ts
diff --git a/...ions/wait_for_index_status_yellow.test.ts → ...ions/wait_for_index_status_yellow.test.ts b/...ions/wait_for_index_status_yellow.test.ts → ...ions/wait_for_index_status_yellow.test.ts
diff --git a/...2/actions/wait_for_index_status_yellow.ts → ...s/actions/wait_for_index_status_yellow.ts b/...2/actions/wait_for_index_status_yellow.ts → ...s/actions/wait_for_index_status_yellow.ts
diff --git a/..._for_pickup_updated_mappings_task.test.ts → ..._for_pickup_updated_mappings_task.test.ts b/..._for_pickup_updated_mappings_task.test.ts → ..._for_pickup_updated_mappings_task.test.ts
diff --git a/.../wait_for_pickup_updated_mappings_task.ts → .../wait_for_pickup_updated_mappings_task.ts b/.../wait_for_pickup_updated_mappings_task.ts → .../wait_for_pickup_updated_mappings_task.ts
diff --git a/...sv2/actions/wait_for_reindex_task.test.ts → ...ons/actions/wait_for_reindex_task.test.ts b/...sv2/actions/wait_for_reindex_task.test.ts → ...ons/actions/wait_for_reindex_task.test.ts
diff --git a/...ationsv2/actions/wait_for_reindex_task.ts → ...grations/actions/wait_for_reindex_task.ts b/...ationsv2/actions/wait_for_reindex_task.ts → ...grations/actions/wait_for_reindex_task.ts
diff --git a/...igrationsv2/actions/wait_for_task.test.ts → .../migrations/actions/wait_for_task.test.ts b/...igrationsv2/actions/wait_for_task.test.ts → .../migrations/actions/wait_for_task.test.ts
diff --git a/...cts/migrationsv2/actions/wait_for_task.ts → ...jects/migrations/actions/wait_for_task.ts b/...cts/migrationsv2/actions/wait_for_task.ts → ...jects/migrations/actions/wait_for_task.ts
diff --git a/src/core/server/saved_objects/migrations/core/__snapshots__/elastic_index.test.ts.snap b/src/core/server/saved_objects/migrations/core/__snapshots__/elastic_index.test.ts.snap
diff --git a/src/core/server/saved_objects/migrations/core/call_cluster.ts b/src/core/server/saved_objects/migrations/core/call_cluster.ts
diff --git a/...migrations/core/migration_context.test.ts → ...sable_unknown_type_mapping_fields.test.ts b/...migrations/core/migration_context.test.ts → ...sable_unknown_type_mapping_fields.test.ts
@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-import { disableUnknownTypeMappingFields } from './migration_context';
+import { disableUnknownTypeMappingFields } from './disable_unknown_type_mapping_fields';
 
 describe('disableUnknownTypeMappingFields', () => {
   const sourceMappings = {

diff --git a/src/core/server/saved_objects/migrations/core/disable_unknown_type_mapping_fields.ts b/src/core/server/saved_objects/migrations/core/disable_unknown_type_mapping_fields.ts
@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { SavedObjectsMappingProperties, IndexMapping } from '../../mappings';
+
+/**
+ * Merges the active mappings and the source mappings while disabling the
+ * fields of any unknown Saved Object types present in the source index's
+ * mappings.
+ *
+ * Since the Saved Objects index has `dynamic: strict` defined at the
+ * top-level, only Saved Object types for which a mapping exists can be
+ * inserted into the index. To ensure that we can continue to store Saved
+ * Object documents belonging to a disabled plugin we define a mapping for all
+ * the unknown Saved Object types that were present in the source index's
+ * mappings. To limit the field count as much as possible, these unkwnown
+ * type's mappings are set to `dynamic: false`.
+ *
+ * (Since we're using the source index mappings instead of looking at actual
+ * document types in the inedx, we potentially add more "unknown types" than
+ * what would be necessary to support migrating all the data over to the
+ * target index.)
+ *
+ * @param activeMappings The mappings compiled from all the Saved Object types
+ * known to this Kibana node.
+ * @param sourceMappings The mappings of index used as the migration source.
+ * @returns The mappings that should be applied to the target index.
+ */
+export function disableUnknownTypeMappingFields(
+  activeMappings: IndexMapping,
+  sourceMappings: IndexMapping
+): IndexMapping {
+  const targetTypes = Object.keys(activeMappings.properties);
+
+  const disabledTypesProperties = Object.keys(sourceMappings.properties ?? {})
+    .filter((sourceType) => {
+      const isObjectType = 'properties' in sourceMappings.properties[sourceType];
+      // Only Object/Nested datatypes can be excluded from the field count by
+      // using `dynamic: false`.
+      return !targetTypes.includes(sourceType) && isObjectType;
+    })
+    .reduce((disabledTypesAcc, sourceType) => {
+      disabledTypesAcc[sourceType] = { dynamic: false, properties: {} };
+      return disabledTypesAcc;
+    }, {} as SavedObjectsMappingProperties);
+
+  return {
+    ...activeMappings,
+    properties: {
+      ...sourceMappings.properties,
+      ...disabledTypesProperties,
+      ...activeMappings.properties,
+    },
+  };
+}