From 8cd0a03d1ceb3011af8d7cebf5592d41f53f5dbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Loix?= Date: Fri, 31 Jan 2020 16:24:31 +0530 Subject: [PATCH] [7.5] [Index template] Fix editor should support mappings types (#55804) (#56279) --- .../components/mappings_editor/index.ts | 7 + .../mappings_editor/lib/error_reporter.ts | 42 ++++++ .../lib/extract_mappings_definition.ts | 56 +++++++ .../components/mappings_editor/lib/index.ts | 9 ++ .../mappings_editor/lib/mappings_validator.ts | 106 +++++++++++++ .../components/mappings_editor/types.ts | 142 ++++++++++++++++++ .../index_management/public/services/api.ts | 7 +- .../api/mapping/register_mapping_route.js | 1 + .../api/templates/register_create_route.ts | 2 + .../api/templates/register_get_routes.ts | 9 +- .../api/templates/register_update_route.ts | 2 + .../management/index_management/mapping.js | 3 +- 12 files changed, 381 insertions(+), 5 deletions(-) create mode 100644 x-pack/legacy/plugins/index_management/public/components/mappings_editor/index.ts create mode 100644 x-pack/legacy/plugins/index_management/public/components/mappings_editor/lib/error_reporter.ts create mode 100644 x-pack/legacy/plugins/index_management/public/components/mappings_editor/lib/extract_mappings_definition.ts create mode 100644 x-pack/legacy/plugins/index_management/public/components/mappings_editor/lib/index.ts create mode 100644 x-pack/legacy/plugins/index_management/public/components/mappings_editor/lib/mappings_validator.ts create mode 100644 x-pack/legacy/plugins/index_management/public/components/mappings_editor/types.ts diff --git a/x-pack/legacy/plugins/index_management/public/components/mappings_editor/index.ts b/x-pack/legacy/plugins/index_management/public/components/mappings_editor/index.ts new file mode 100644 index 0000000000000..17555951fc027 --- /dev/null +++ b/x-pack/legacy/plugins/index_management/public/components/mappings_editor/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { doMappingsHaveType } from './lib'; diff --git a/x-pack/legacy/plugins/index_management/public/components/mappings_editor/lib/error_reporter.ts b/x-pack/legacy/plugins/index_management/public/components/mappings_editor/lib/error_reporter.ts new file mode 100644 index 0000000000000..e9beee1071597 --- /dev/null +++ b/x-pack/legacy/plugins/index_management/public/components/mappings_editor/lib/error_reporter.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { ValidationError } from 'io-ts'; +import { fold } from 'fp-ts/lib/Either'; +import { Reporter } from 'io-ts/lib/Reporter'; + +export type ReporterResult = Array<{ path: string[]; message: string }>; + +const failure = (validation: ValidationError[]): ReporterResult => { + return validation.map(e => { + const path: string[] = []; + let validationName = ''; + + e.context.forEach((ctx, idx) => { + if (ctx.key) { + path.push(ctx.key); + } + + if (idx === e.context.length - 1) { + validationName = ctx.type.name; + } + }); + const lastItemName = path[path.length - 1]; + return { + path, + message: + 'Invalid value ' + + JSON.stringify(e.value) + + ` supplied to ${lastItemName}(${validationName})`, + }; + }); +}; + +const empty: never[] = []; +const success = () => empty; + +export const errorReporter: Reporter = { + report: fold(failure, success), +}; diff --git a/x-pack/legacy/plugins/index_management/public/components/mappings_editor/lib/extract_mappings_definition.ts b/x-pack/legacy/plugins/index_management/public/components/mappings_editor/lib/extract_mappings_definition.ts new file mode 100644 index 0000000000000..a566f466edaf9 --- /dev/null +++ b/x-pack/legacy/plugins/index_management/public/components/mappings_editor/lib/extract_mappings_definition.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { isPlainObject } from 'lodash'; + +import { GenericObject } from '../types'; +import { validateMappingsConfiguration, VALID_MAPPINGS_PARAMETERS } from './mappings_validator'; + +interface MappingsWithType { + type?: string; + mappings: GenericObject; +} + +const isMappingDefinition = (obj: GenericObject): boolean => { + const areAllKeysValid = Object.keys(obj).every(key => VALID_MAPPINGS_PARAMETERS.includes(key)); + + if (!areAllKeysValid) { + return false; + } + + const { properties, dynamic_templates: dynamicTemplates, ...mappingsConfiguration } = obj; + + const { errors } = validateMappingsConfiguration(mappingsConfiguration); + const isConfigurationValid = errors.length === 0; + const isPropertiesValid = properties === undefined || isPlainObject(properties); + const isDynamicTemplatesValid = dynamicTemplates === undefined || Array.isArray(dynamicTemplates); + + // If the configuration, the properties and the dynamic templates are valid + // we can assume that the mapping is declared at root level (no types) + return isConfigurationValid && isPropertiesValid && isDynamicTemplatesValid; +}; + +const getMappingsDefinitionWithType = (mappings: GenericObject): MappingsWithType[] => { + if (isMappingDefinition(mappings)) { + // No need to go any further + return [{ mappings }]; + } + + // At this point there must be one or more type mappings + const typedMappings = Object.entries(mappings).reduce( + (acc: Array<{ type: string; mappings: GenericObject }>, [type, value]) => { + if (isMappingDefinition(value)) { + acc.push({ type, mappings: value as GenericObject }); + } + return acc; + }, + [] + ); + + return typedMappings; +}; + +export const doMappingsHaveType = (mappings: GenericObject = {}): boolean => + getMappingsDefinitionWithType(mappings).filter(({ type }) => type !== undefined).length > 0; diff --git a/x-pack/legacy/plugins/index_management/public/components/mappings_editor/lib/index.ts b/x-pack/legacy/plugins/index_management/public/components/mappings_editor/lib/index.ts new file mode 100644 index 0000000000000..857576440dfa4 --- /dev/null +++ b/x-pack/legacy/plugins/index_management/public/components/mappings_editor/lib/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './mappings_validator'; + +export * from './extract_mappings_definition'; diff --git a/x-pack/legacy/plugins/index_management/public/components/mappings_editor/lib/mappings_validator.ts b/x-pack/legacy/plugins/index_management/public/components/mappings_editor/lib/mappings_validator.ts new file mode 100644 index 0000000000000..95bbb6b07ad35 --- /dev/null +++ b/x-pack/legacy/plugins/index_management/public/components/mappings_editor/lib/mappings_validator.ts @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { pick } from 'lodash'; +import * as t from 'io-ts'; +import { ordString } from 'fp-ts/lib/Ord'; +import { toArray } from 'fp-ts/lib/Set'; +import { isLeft } from 'fp-ts/lib/Either'; + +import { errorReporter } from './error_reporter'; + +type MappingsValidationError = + | { code: 'ERR_CONFIG'; configName: string } + | { code: 'ERR_FIELD'; fieldPath: string } + | { code: 'ERR_PARAMETER'; paramName: string; fieldPath: string }; + +/** + * Single source of truth to validate the *configuration* of the mappings. + * Whenever a user loads a JSON object it will be validate against this Joi schema. + */ +const mappingsConfigurationSchema = t.exact( + t.partial({ + dynamic: t.union([t.literal(true), t.literal(false), t.literal('strict')]), + date_detection: t.boolean, + numeric_detection: t.boolean, + dynamic_date_formats: t.array(t.string), + _source: t.exact( + t.partial({ + enabled: t.boolean, + includes: t.array(t.string), + excludes: t.array(t.string), + }) + ), + _meta: t.UnknownRecord, + _routing: t.interface({ + required: t.boolean, + }), + }) +); + +const mappingsConfigurationSchemaKeys = Object.keys(mappingsConfigurationSchema.type.props); +const sourceConfigurationSchemaKeys = Object.keys( + mappingsConfigurationSchema.type.props._source.type.props +); + +export const validateMappingsConfiguration = ( + mappingsConfiguration: any +): { value: any; errors: MappingsValidationError[] } => { + // Set to keep track of invalid configuration parameters. + const configurationRemoved: Set = new Set(); + + let copyOfMappingsConfig = { ...mappingsConfiguration }; + const result = mappingsConfigurationSchema.decode(mappingsConfiguration); + const isSchemaInvalid = isLeft(result); + + const unknownConfigurationParameters = Object.keys(mappingsConfiguration).filter( + key => mappingsConfigurationSchemaKeys.includes(key) === false + ); + + const unknownSourceConfigurationParameters = + mappingsConfiguration._source !== undefined + ? Object.keys(mappingsConfiguration._source).filter( + key => sourceConfigurationSchemaKeys.includes(key) === false + ) + : []; + + if (isSchemaInvalid) { + /** + * To keep the logic simple we will strip out the parameters that contain errors + */ + const errors = errorReporter.report(result); + errors.forEach(error => { + const configurationName = error.path[0]; + configurationRemoved.add(configurationName); + delete copyOfMappingsConfig[configurationName]; + }); + } + + if (unknownConfigurationParameters.length > 0) { + unknownConfigurationParameters.forEach(configName => configurationRemoved.add(configName)); + } + + if (unknownSourceConfigurationParameters.length > 0) { + configurationRemoved.add('_source'); + delete copyOfMappingsConfig._source; + } + + copyOfMappingsConfig = pick(copyOfMappingsConfig, mappingsConfigurationSchemaKeys); + + const errors: MappingsValidationError[] = toArray(ordString)(configurationRemoved) + .map(configName => ({ + code: 'ERR_CONFIG', + configName, + })) + .sort((a, b) => a.configName.localeCompare(b.configName)) as MappingsValidationError[]; + + return { value: copyOfMappingsConfig, errors }; +}; + +export const VALID_MAPPINGS_PARAMETERS = [ + ...mappingsConfigurationSchemaKeys, + 'dynamic_templates', + 'properties', +]; diff --git a/x-pack/legacy/plugins/index_management/public/components/mappings_editor/types.ts b/x-pack/legacy/plugins/index_management/public/components/mappings_editor/types.ts new file mode 100644 index 0000000000000..a4a6fef10ee6c --- /dev/null +++ b/x-pack/legacy/plugins/index_management/public/components/mappings_editor/types.ts @@ -0,0 +1,142 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { ReactNode } from 'react'; + +export interface DataTypeDefinition { + label: string; + value: DataType; + documentation?: { + main: string; + [key: string]: string; + }; + subTypes?: { label: string; types: SubType[] }; + description?: () => ReactNode; +} + +export type MainType = + | 'text' + | 'keyword' + | 'numeric' + | 'binary' + | 'boolean' + | 'range' + | 'object' + | 'nested' + | 'alias' + | 'completion' + | 'dense_vector' + | 'flattened' + | 'ip' + | 'join' + | 'percolator' + | 'rank_feature' + | 'rank_features' + | 'shape' + | 'search_as_you_type' + | 'date' + | 'date_nanos' + | 'geo_point' + | 'geo_shape' + | 'token_count'; + +export type SubType = NumericType | RangeType; + +export type DataType = MainType | SubType; + +export type NumericType = + | 'long' + | 'integer' + | 'short' + | 'byte' + | 'double' + | 'float' + | 'half_float' + | 'scaled_float'; + +export type RangeType = + | 'integer_range' + | 'float_range' + | 'long_range' + | 'ip_range' + | 'double_range' + | 'date_range'; + +export type ParameterName = + | 'name' + | 'type' + | 'store' + | 'index' + | 'fielddata' + | 'fielddata_frequency_filter' + | 'fielddata_frequency_filter_percentage' + | 'fielddata_frequency_filter_absolute' + | 'doc_values' + | 'doc_values_binary' + | 'coerce' + | 'coerce_shape' + | 'ignore_malformed' + | 'null_value' + | 'null_value_numeric' + | 'null_value_boolean' + | 'null_value_geo_point' + | 'null_value_ip' + | 'copy_to' + | 'dynamic' + | 'dynamic_toggle' + | 'dynamic_strict' + | 'enabled' + | 'boost' + | 'locale' + | 'format' + | 'analyzer' + | 'search_analyzer' + | 'search_quote_analyzer' + | 'index_options' + | 'index_options_flattened' + | 'index_options_keyword' + | 'eager_global_ordinals' + | 'eager_global_ordinals_join' + | 'index_prefixes' + | 'index_phrases' + | 'norms' + | 'norms_keyword' + | 'term_vector' + | 'position_increment_gap' + | 'similarity' + | 'normalizer' + | 'ignore_above' + | 'split_queries_on_whitespace' + | 'scaling_factor' + | 'max_input_length' + | 'preserve_separators' + | 'preserve_position_increments' + | 'ignore_z_value' + | 'enable_position_increments' + | 'orientation' + | 'points_only' + | 'path' + | 'dims' + | 'depth_limit' + | 'relations' + | 'max_shingle_size'; + +interface FieldBasic { + name: string; + type: DataType; + subType?: SubType; + properties?: { [key: string]: Omit }; + fields?: { [key: string]: Omit }; +} + +type FieldParams = { + [K in ParameterName]: unknown; +}; + +export type Field = FieldBasic & Partial; + +export interface GenericObject { + [key: string]: any; +} diff --git a/x-pack/legacy/plugins/index_management/public/services/api.ts b/x-pack/legacy/plugins/index_management/public/services/api.ts index 4592381cad631..cf0e078f6b27f 100644 --- a/x-pack/legacy/plugins/index_management/public/services/api.ts +++ b/x-pack/legacy/plugins/index_management/public/services/api.ts @@ -38,6 +38,7 @@ import { TAB_SETTINGS, TAB_MAPPING, TAB_STATS } from '../constants'; import { trackUiMetric, METRIC_TYPE } from './track_ui_metric'; import { useRequest, sendRequest } from './use_request'; import { Template } from '../../common/types'; +import { doMappingsHaveType } from '../components/mappings_editor'; let httpClient: ng.IHttpService; @@ -225,8 +226,9 @@ export function loadIndexTemplate(name: Template['name']) { } export async function saveTemplate(template: Template, isClone?: boolean) { + const includeTypeName = doMappingsHaveType(template.mappings); const result = sendRequest({ - path: `${apiPrefix}/templates`, + path: `${apiPrefix}/templates?include_type_name=${includeTypeName}`, method: 'put', body: template, }); @@ -239,9 +241,10 @@ export async function saveTemplate(template: Template, isClone?: boolean) { } export async function updateTemplate(template: Template) { + const includeTypeName = doMappingsHaveType(template.mappings); const { name } = template; const result = sendRequest({ - path: `${apiPrefix}/templates/${encodeURIComponent(name)}`, + path: `${apiPrefix}/templates/${encodeURIComponent(name)}?include_type_name=${includeTypeName}`, method: 'put', body: template, }); diff --git a/x-pack/legacy/plugins/index_management/server/routes/api/mapping/register_mapping_route.js b/x-pack/legacy/plugins/index_management/server/routes/api/mapping/register_mapping_route.js index 790aa21bf9b84..94c36af776d15 100644 --- a/x-pack/legacy/plugins/index_management/server/routes/api/mapping/register_mapping_route.js +++ b/x-pack/legacy/plugins/index_management/server/routes/api/mapping/register_mapping_route.js @@ -15,6 +15,7 @@ const handler = async (request, callWithRequest) => { const params = { expand_wildcards: 'none', index: indexName, + include_type_name: true, }; const hit = await callWithRequest('indices.getMapping', params); diff --git a/x-pack/legacy/plugins/index_management/server/routes/api/templates/register_create_route.ts b/x-pack/legacy/plugins/index_management/server/routes/api/templates/register_create_route.ts index e134a97dd029e..8cc9f24f78a52 100644 --- a/x-pack/legacy/plugins/index_management/server/routes/api/templates/register_create_route.ts +++ b/x-pack/legacy/plugins/index_management/server/routes/api/templates/register_create_route.ts @@ -15,6 +15,7 @@ import { serializeTemplate } from '../../../../common/lib'; const handler: RouterRouteHandler = async (req, callWithRequest) => { const template = req.payload as Template; + const { include_type_name } = req.query as any; const serializedTemplate = serializeTemplate(template) as TemplateEs; const { name, order, index_patterns, version, settings, mappings, aliases } = serializedTemplate; @@ -49,6 +50,7 @@ const handler: RouterRouteHandler = async (req, callWithRequest) => { return await callWithRequest('indices.putTemplate', { name, order, + include_type_name, body: { index_patterns, version, diff --git a/x-pack/legacy/plugins/index_management/server/routes/api/templates/register_get_routes.ts b/x-pack/legacy/plugins/index_management/server/routes/api/templates/register_get_routes.ts index b450f75d1cc53..555feafa053d1 100644 --- a/x-pack/legacy/plugins/index_management/server/routes/api/templates/register_get_routes.ts +++ b/x-pack/legacy/plugins/index_management/server/routes/api/templates/register_get_routes.ts @@ -13,7 +13,9 @@ let callWithInternalUser: any; const allHandler: RouterRouteHandler = async (_req, callWithRequest) => { const managedTemplatePrefix = await getManagedTemplatePrefix(callWithInternalUser); - const indexTemplatesByName = await callWithRequest('indices.getTemplate'); + const indexTemplatesByName = await callWithRequest('indices.getTemplate', { + include_type_name: true, + }); return deserializeTemplateList(indexTemplatesByName, managedTemplatePrefix); }; @@ -21,7 +23,10 @@ const allHandler: RouterRouteHandler = async (_req, callWithRequest) => { const oneHandler: RouterRouteHandler = async (req, callWithRequest) => { const { name } = req.params; const managedTemplatePrefix = await getManagedTemplatePrefix(callWithInternalUser); - const indexTemplateByName = await callWithRequest('indices.getTemplate', { name }); + const indexTemplateByName = await callWithRequest('indices.getTemplate', { + name, + include_type_name: true, + }); if (indexTemplateByName[name]) { return deserializeTemplate({ ...indexTemplateByName[name], name }, managedTemplatePrefix); diff --git a/x-pack/legacy/plugins/index_management/server/routes/api/templates/register_update_route.ts b/x-pack/legacy/plugins/index_management/server/routes/api/templates/register_update_route.ts index 15590e2acbe71..ccbda2dab9364 100644 --- a/x-pack/legacy/plugins/index_management/server/routes/api/templates/register_update_route.ts +++ b/x-pack/legacy/plugins/index_management/server/routes/api/templates/register_update_route.ts @@ -10,6 +10,7 @@ import { serializeTemplate } from '../../../../common/lib'; const handler: RouterRouteHandler = async (req, callWithRequest) => { const { name } = req.params; + const { include_type_name } = req.query as any; const template = req.payload as Template; const serializedTemplate = serializeTemplate(template) as TemplateEs; @@ -22,6 +23,7 @@ const handler: RouterRouteHandler = async (req, callWithRequest) => { return await callWithRequest('indices.putTemplate', { name, order, + include_type_name, body: { index_patterns, version, diff --git a/x-pack/test/api_integration/apis/management/index_management/mapping.js b/x-pack/test/api_integration/apis/management/index_management/mapping.js index fa0f6e04a7a4d..13eca5bef251a 100644 --- a/x-pack/test/api_integration/apis/management/index_management/mapping.js +++ b/x-pack/test/api_integration/apis/management/index_management/mapping.js @@ -32,7 +32,8 @@ export default function({ getService }) { const { body } = await getIndexMapping(index).expect(200); - expect(body.mapping).to.eql(mappings); + // As, on 7.x we require the mappings with type (include_type_name), the default "_doc" type is returned + expect(body.mapping).to.eql({ _doc: mappings }); }); }); }