From e3877e053405bae71b5576648cb7c637c4a23f9a Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 16 Dec 2024 21:58:30 +0100 Subject: [PATCH] Fix generation of dynamic mapping for object with specific subfield (#204104) Fix generation of dynamic mapping for objects that have more specific subfields in separate definitions. This can be reproduced for example with: ``` - name: labels type: object object_type: keyword object_type_mapping_type: '*' - name: labels.count type: long ``` Fleet expands and deduplicates field definitions before generating the mappings, so the definitions above are converted to something like the following: ``` - name: labels type: group object_type: keyword object_type_mapping_type: '*' fields: - name: count type: long ``` Usually fields of type `group` don't have an `object_type`, so this was being ignored, the dynamic mapping was not being generated. This issue was not reproduced if the object field name includes a wildcard, like in `labels.*`, because then the expansion and deduplication resolves to something like this: ``` - name: labels type: group fields: - name: '*' type: object object_type: keyword object_type_mapping_type: '*' - name: count type: long ``` --- .../elasticsearch/template/template.test.ts | 78 ++++++ .../epm/elasticsearch/template/template.ts | 242 +++++++++--------- 2 files changed, 203 insertions(+), 117 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index 9fc5383902ff3..e42c92b7f0cd4 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -816,6 +816,84 @@ describe('EPM template', () => { expect(mappings).toEqual(objectFieldWithPropertyReversedMapping); }); + it('tests processing object field with more specific properties without wildcard', () => { + const objectFieldWithPropertyReversedLiteralYml = ` +- name: labels + type: object + object_type: keyword + object_type_mapping_type: '*' +- name: labels.count + type: long +`; + const objectFieldWithPropertyReversedMapping = { + dynamic_templates: [ + { + labels: { + path_match: 'labels.*', + match_mapping_type: '*', + mapping: { + type: 'keyword', + }, + }, + }, + ], + properties: { + labels: { + dynamic: true, + type: 'object', + properties: { + count: { + type: 'long', + }, + }, + }, + }, + }; + const fields: Field[] = load(objectFieldWithPropertyReversedLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(objectFieldWithPropertyReversedMapping); + }); + + it('tests processing object field with more specific properties with wildcard', () => { + const objectFieldWithPropertyReversedLiteralYml = ` +- name: labels.* + type: object + object_type: keyword + object_type_mapping_type: '*' +- name: labels.count + type: long +`; + const objectFieldWithPropertyReversedMapping = { + dynamic_templates: [ + { + 'labels.*': { + path_match: 'labels.*', + match_mapping_type: '*', + mapping: { + type: 'keyword', + }, + }, + }, + ], + properties: { + labels: { + dynamic: true, + type: 'object', + properties: { + count: { + type: 'long', + }, + }, + }, + }, + }; + const fields: Field[] = load(objectFieldWithPropertyReversedLiteralYml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(objectFieldWithPropertyReversedMapping); + }); + it('tests processing object field with subobjects set to false (case B)', () => { const objectFieldWithPropertyReversedLiteralYml = ` - name: b.labels.* diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index d44cd57a2c5ba..93695c68add0a 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -310,6 +310,124 @@ function _generateMappings( } } + function addObjectAsDynamicMapping(field: Field) { + const path = ctx.groupFieldName ? `${ctx.groupFieldName}.${field.name}` : field.name; + const pathMatch = path.includes('*') ? path : `${path}.*`; + + let dynProperties: Properties = getDefaultProperties(field); + let matchingType: string | undefined; + switch (field.object_type) { + case 'histogram': + dynProperties = histogram(field); + matchingType = field.object_type_mapping_type ?? '*'; + break; + case 'ip': + case 'keyword': + case 'match_only_text': + case 'text': + case 'wildcard': + dynProperties.type = field.object_type; + matchingType = field.object_type_mapping_type ?? 'string'; + break; + case 'scaled_float': + dynProperties = scaledFloat(field); + matchingType = field.object_type_mapping_type ?? '*'; + break; + case 'aggregate_metric_double': + dynProperties.type = field.object_type; + dynProperties.metrics = field.metrics; + dynProperties.default_metric = field.default_metric; + matchingType = field.object_type_mapping_type ?? '*'; + break; + case 'double': + case 'float': + case 'half_float': + dynProperties.type = field.object_type; + if (isIndexModeTimeSeries) { + dynProperties.time_series_metric = field.metric_type; + } + matchingType = field.object_type_mapping_type ?? 'double'; + break; + case 'byte': + case 'long': + case 'short': + case 'unsigned_long': + dynProperties.type = field.object_type; + if (isIndexModeTimeSeries) { + dynProperties.time_series_metric = field.metric_type; + } + matchingType = field.object_type_mapping_type ?? 'long'; + break; + case 'integer': + // Map integers as long, as in other cases. + dynProperties.type = 'long'; + if (isIndexModeTimeSeries) { + dynProperties.time_series_metric = field.metric_type; + } + matchingType = field.object_type_mapping_type ?? 'long'; + break; + case 'boolean': + dynProperties.type = field.object_type; + if (isIndexModeTimeSeries) { + dynProperties.time_series_metric = field.metric_type; + } + matchingType = field.object_type_mapping_type ?? field.object_type; + break; + case 'group': + if (!field?.fields) { + break; + } + const subFields = field.fields.map((subField) => ({ + ...subField, + type: 'object', + object_type: subField.object_type ?? subField.type, + })); + const mappings = _generateMappings( + subFields, + { + ...ctx, + groupFieldName: ctx.groupFieldName ? `${ctx.groupFieldName}.${field.name}` : field.name, + }, + isIndexModeTimeSeries + ); + if (mappings.hasDynamicTemplateMappings) { + hasDynamicTemplateMappings = true; + } + break; + case 'flattened': + dynProperties.type = field.object_type; + matchingType = field.object_type_mapping_type ?? 'object'; + break; + default: + throw new PackageInvalidArchiveError( + `No dynamic mapping generated for field ${path} of type ${field.object_type}` + ); + } + + if (field.dimension && isIndexModeTimeSeries) { + dynProperties.time_series_dimension = field.dimension; + } + + // When a wildcard field specifies the subobjects setting, + // the parent intermediate object should set the subobjects + // setting. + // + // For example, if a wildcard field `foo.*` has subobjects, + // we should set subobjects on the intermediate object `foo`. + // + if (field.subobjects !== undefined && path.includes('*')) { + subobjects = field.subobjects; + } + + if (dynProperties && matchingType) { + addDynamicMappingWithIntermediateObjects(path, pathMatch, matchingType, dynProperties); + + // Add the parent object as static property, this is needed for + // index templates not using `"dynamic": true`. + addParentObjectAsStaticProperty(field); + } + } + // TODO: this can happen when the fields property in fields.yml is present but empty // Maybe validation should be moved to fields/field.ts if (fields) { @@ -371,123 +489,7 @@ function _generateMappings( } if (type === 'object' && field.object_type) { - const path = ctx.groupFieldName ? `${ctx.groupFieldName}.${field.name}` : field.name; - const pathMatch = path.includes('*') ? path : `${path}.*`; - - let dynProperties: Properties = getDefaultProperties(field); - let matchingType: string | undefined; - switch (field.object_type) { - case 'histogram': - dynProperties = histogram(field); - matchingType = field.object_type_mapping_type ?? '*'; - break; - case 'ip': - case 'keyword': - case 'match_only_text': - case 'text': - case 'wildcard': - dynProperties.type = field.object_type; - matchingType = field.object_type_mapping_type ?? 'string'; - break; - case 'scaled_float': - dynProperties = scaledFloat(field); - matchingType = field.object_type_mapping_type ?? '*'; - break; - case 'aggregate_metric_double': - dynProperties.type = field.object_type; - dynProperties.metrics = field.metrics; - dynProperties.default_metric = field.default_metric; - matchingType = field.object_type_mapping_type ?? '*'; - break; - case 'double': - case 'float': - case 'half_float': - dynProperties.type = field.object_type; - if (isIndexModeTimeSeries) { - dynProperties.time_series_metric = field.metric_type; - } - matchingType = field.object_type_mapping_type ?? 'double'; - break; - case 'byte': - case 'long': - case 'short': - case 'unsigned_long': - dynProperties.type = field.object_type; - if (isIndexModeTimeSeries) { - dynProperties.time_series_metric = field.metric_type; - } - matchingType = field.object_type_mapping_type ?? 'long'; - break; - case 'integer': - // Map integers as long, as in other cases. - dynProperties.type = 'long'; - if (isIndexModeTimeSeries) { - dynProperties.time_series_metric = field.metric_type; - } - matchingType = field.object_type_mapping_type ?? 'long'; - break; - case 'boolean': - dynProperties.type = field.object_type; - if (isIndexModeTimeSeries) { - dynProperties.time_series_metric = field.metric_type; - } - matchingType = field.object_type_mapping_type ?? field.object_type; - break; - case 'group': - if (!field?.fields) { - break; - } - const subFields = field.fields.map((subField) => ({ - ...subField, - type: 'object', - object_type: subField.object_type ?? subField.type, - })); - const mappings = _generateMappings( - subFields, - { - ...ctx, - groupFieldName: ctx.groupFieldName - ? `${ctx.groupFieldName}.${field.name}` - : field.name, - }, - isIndexModeTimeSeries - ); - if (mappings.hasDynamicTemplateMappings) { - hasDynamicTemplateMappings = true; - } - break; - case 'flattened': - dynProperties.type = field.object_type; - matchingType = field.object_type_mapping_type ?? 'object'; - break; - default: - throw new PackageInvalidArchiveError( - `No dynamic mapping generated for field ${path} of type ${field.object_type}` - ); - } - - if (field.dimension && isIndexModeTimeSeries) { - dynProperties.time_series_dimension = field.dimension; - } - - // When a wildcard field specifies the subobjects setting, - // the parent intermediate object should set the subobjects - // setting. - // - // For example, if a wildcard field `foo.*` has subobjects, - // we should set subobjects on the intermediate object `foo`. - // - if (field.subobjects !== undefined && path.includes('*')) { - subobjects = field.subobjects; - } - - if (dynProperties && matchingType) { - addDynamicMappingWithIntermediateObjects(path, pathMatch, matchingType, dynProperties); - - // Add the parent object as static property, this is needed for - // index templates not using `"dynamic": true`. - addParentObjectAsStaticProperty(field); - } + addObjectAsDynamicMapping(field); } else { let fieldProps = getDefaultProperties(field); @@ -503,6 +505,12 @@ function _generateMappings( }, isIndexModeTimeSeries ); + if (field.object_type) { + // A group can have an object_type if it has been merged with an object during deduplication, + // generate also the dynamic mapping for the object. + addObjectAsDynamicMapping(field); + mappings.hasDynamicTemplateMappings = true; + } if (mappings.hasNonDynamicTemplateMappings) { fieldProps = { properties: