Skip to content

Commit

Permalink
Fix generation of dynamic mapping for object with specific subfield (e…
Browse files Browse the repository at this point in the history 
…lastic#204104)

Fix generation of dynamic mapping for objects that have more specific
subfields in separate definitions.

This can be reproduced for example with:
```
- name: labels
  type: object
  object_type: keyword
  object_type_mapping_type: '*'
- name: labels.count
  type: long
```

Fleet expands and deduplicates field definitions before generating the
mappings, so the definitions above are converted to something like the
following:
```
- name: labels
  type: group
  object_type: keyword
  object_type_mapping_type: '*'
  fields:
  - name: count
    type: long
```

Usually fields of type `group` don't have an `object_type`, so this was
being ignored, the dynamic mapping was not being generated.

This issue was not reproduced if the object field name includes a
wildcard, like in `labels.*`, because then the expansion and
deduplication resolves to something like this:
```
- name: labels
  type: group
  fields:
  - name: '*'
    type: object
    object_type: keyword
    object_type_mapping_type: '*'
  - name: count
    type: long
```
  • Loading branch information
@jsoriano
jsoriano authored Dec 16, 2024
1 parent 05f2cba commit e3877e0
Show file tree
Hide file tree
Showing 2 changed files with 203 additions and 117 deletions.
78 changes: 78 additions & 0 deletions x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -816,6 +816,84 @@ describe('EPM template', () => {
expect(mappings).toEqual(objectFieldWithPropertyReversedMapping);
});

it('tests processing object field with more specific properties without wildcard', () => {
const objectFieldWithPropertyReversedLiteralYml = `
- name: labels
type: object
object_type: keyword
object_type_mapping_type: '*'
- name: labels.count
type: long
`;
const objectFieldWithPropertyReversedMapping = {
dynamic_templates: [
{
labels: {
path_match: 'labels.*',
match_mapping_type: '*',
mapping: {
type: 'keyword',
},
},
},
],
properties: {
labels: {
dynamic: true,
type: 'object',
properties: {
count: {
type: 'long',
},
},
},
},
};
const fields: Field[] = load(objectFieldWithPropertyReversedLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(mappings).toEqual(objectFieldWithPropertyReversedMapping);
});

it('tests processing object field with more specific properties with wildcard', () => {
const objectFieldWithPropertyReversedLiteralYml = `
- name: labels.*
type: object
object_type: keyword
object_type_mapping_type: '*'
- name: labels.count
type: long
`;
const objectFieldWithPropertyReversedMapping = {
dynamic_templates: [
{
'labels.*': {
path_match: 'labels.*',
match_mapping_type: '*',
mapping: {
type: 'keyword',
},
},
},
],
properties: {
labels: {
dynamic: true,
type: 'object',
properties: {
count: {
type: 'long',
},
},
},
},
};
const fields: Field[] = load(objectFieldWithPropertyReversedLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(mappings).toEqual(objectFieldWithPropertyReversedMapping);
});

it('tests processing object field with subobjects set to false (case B)', () => {
const objectFieldWithPropertyReversedLiteralYml = `
- name: b.labels.*
Expand Down
242 changes: 125 additions & 117 deletions x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -310,6 +310,124 @@ function _generateMappings(
}
}

function addObjectAsDynamicMapping(field: Field) {
const path = ctx.groupFieldName ? `${ctx.groupFieldName}.${field.name}` : field.name;
const pathMatch = path.includes('*') ? path : `${path}.*`;

let dynProperties: Properties = getDefaultProperties(field);
let matchingType: string | undefined;
switch (field.object_type) {
case 'histogram':
dynProperties = histogram(field);
matchingType = field.object_type_mapping_type ?? '*';
break;
case 'ip':
case 'keyword':
case 'match_only_text':
case 'text':
case 'wildcard':
dynProperties.type = field.object_type;
matchingType = field.object_type_mapping_type ?? 'string';
break;
case 'scaled_float':
dynProperties = scaledFloat(field);
matchingType = field.object_type_mapping_type ?? '*';
break;
case 'aggregate_metric_double':
dynProperties.type = field.object_type;
dynProperties.metrics = field.metrics;
dynProperties.default_metric = field.default_metric;
matchingType = field.object_type_mapping_type ?? '*';
break;
case 'double':
case 'float':
case 'half_float':
dynProperties.type = field.object_type;
if (isIndexModeTimeSeries) {
dynProperties.time_series_metric = field.metric_type;
}
matchingType = field.object_type_mapping_type ?? 'double';
break;
case 'byte':
case 'long':
case 'short':
case 'unsigned_long':
dynProperties.type = field.object_type;
if (isIndexModeTimeSeries) {
dynProperties.time_series_metric = field.metric_type;
}
matchingType = field.object_type_mapping_type ?? 'long';
break;
case 'integer':
// Map integers as long, as in other cases.
dynProperties.type = 'long';
if (isIndexModeTimeSeries) {
dynProperties.time_series_metric = field.metric_type;
}
matchingType = field.object_type_mapping_type ?? 'long';
break;
case 'boolean':
dynProperties.type = field.object_type;
if (isIndexModeTimeSeries) {
dynProperties.time_series_metric = field.metric_type;
}
matchingType = field.object_type_mapping_type ?? field.object_type;
break;
case 'group':
if (!field?.fields) {
break;
}
const subFields = field.fields.map((subField) => ({
...subField,
type: 'object',
object_type: subField.object_type ?? subField.type,
}));
const mappings = _generateMappings(
subFields,
{
...ctx,
groupFieldName: ctx.groupFieldName ? `${ctx.groupFieldName}.${field.name}` : field.name,
},
isIndexModeTimeSeries
);
if (mappings.hasDynamicTemplateMappings) {
hasDynamicTemplateMappings = true;
}
break;
case 'flattened':
dynProperties.type = field.object_type;
matchingType = field.object_type_mapping_type ?? 'object';
break;
default:
throw new PackageInvalidArchiveError(
`No dynamic mapping generated for field ${path} of type ${field.object_type}`
);
}

if (field.dimension && isIndexModeTimeSeries) {
dynProperties.time_series_dimension = field.dimension;
}

// When a wildcard field specifies the subobjects setting,
// the parent intermediate object should set the subobjects
// setting.
//
// For example, if a wildcard field `foo.*` has subobjects,
// we should set subobjects on the intermediate object `foo`.
//
if (field.subobjects !== undefined && path.includes('*')) {
subobjects = field.subobjects;
}

if (dynProperties && matchingType) {
addDynamicMappingWithIntermediateObjects(path, pathMatch, matchingType, dynProperties);

// Add the parent object as static property, this is needed for
// index templates not using `"dynamic": true`.
addParentObjectAsStaticProperty(field);
}
}

// TODO: this can happen when the fields property in fields.yml is present but empty
// Maybe validation should be moved to fields/field.ts
if (fields) {
Expand Down Expand Up @@ -371,123 +489,7 @@ function _generateMappings(
}

if (type === 'object' && field.object_type) {
const path = ctx.groupFieldName ? `${ctx.groupFieldName}.${field.name}` : field.name;
const pathMatch = path.includes('*') ? path : `${path}.*`;

let dynProperties: Properties = getDefaultProperties(field);
let matchingType: string | undefined;
switch (field.object_type) {
case 'histogram':
dynProperties = histogram(field);
matchingType = field.object_type_mapping_type ?? '*';
break;
case 'ip':
case 'keyword':
case 'match_only_text':
case 'text':
case 'wildcard':
dynProperties.type = field.object_type;
matchingType = field.object_type_mapping_type ?? 'string';
break;
case 'scaled_float':
dynProperties = scaledFloat(field);
matchingType = field.object_type_mapping_type ?? '*';
break;
case 'aggregate_metric_double':
dynProperties.type = field.object_type;
dynProperties.metrics = field.metrics;
dynProperties.default_metric = field.default_metric;
matchingType = field.object_type_mapping_type ?? '*';
break;
case 'double':
case 'float':
case 'half_float':
dynProperties.type = field.object_type;
if (isIndexModeTimeSeries) {
dynProperties.time_series_metric = field.metric_type;
}
matchingType = field.object_type_mapping_type ?? 'double';
break;
case 'byte':
case 'long':
case 'short':
case 'unsigned_long':
dynProperties.type = field.object_type;
if (isIndexModeTimeSeries) {
dynProperties.time_series_metric = field.metric_type;
}
matchingType = field.object_type_mapping_type ?? 'long';
break;
case 'integer':
// Map integers as long, as in other cases.
dynProperties.type = 'long';
if (isIndexModeTimeSeries) {
dynProperties.time_series_metric = field.metric_type;
}
matchingType = field.object_type_mapping_type ?? 'long';
break;
case 'boolean':
dynProperties.type = field.object_type;
if (isIndexModeTimeSeries) {
dynProperties.time_series_metric = field.metric_type;
}
matchingType = field.object_type_mapping_type ?? field.object_type;
break;
case 'group':
if (!field?.fields) {
break;
}
const subFields = field.fields.map((subField) => ({
...subField,
type: 'object',
object_type: subField.object_type ?? subField.type,
}));
const mappings = _generateMappings(
subFields,
{
...ctx,
groupFieldName: ctx.groupFieldName
? `${ctx.groupFieldName}.${field.name}`
: field.name,
},
isIndexModeTimeSeries
);
if (mappings.hasDynamicTemplateMappings) {
hasDynamicTemplateMappings = true;
}
break;
case 'flattened':
dynProperties.type = field.object_type;
matchingType = field.object_type_mapping_type ?? 'object';
break;
default:
throw new PackageInvalidArchiveError(
`No dynamic mapping generated for field ${path} of type ${field.object_type}`
);
}

if (field.dimension && isIndexModeTimeSeries) {
dynProperties.time_series_dimension = field.dimension;
}

// When a wildcard field specifies the subobjects setting,
// the parent intermediate object should set the subobjects
// setting.
//
// For example, if a wildcard field `foo.*` has subobjects,
// we should set subobjects on the intermediate object `foo`.
//
if (field.subobjects !== undefined && path.includes('*')) {
subobjects = field.subobjects;
}

if (dynProperties && matchingType) {
addDynamicMappingWithIntermediateObjects(path, pathMatch, matchingType, dynProperties);

// Add the parent object as static property, this is needed for
// index templates not using `"dynamic": true`.
addParentObjectAsStaticProperty(field);
}
addObjectAsDynamicMapping(field);
} else {
let fieldProps = getDefaultProperties(field);

Expand All @@ -503,6 +505,12 @@ function _generateMappings(
},
isIndexModeTimeSeries
);
if (field.object_type) {
// A group can have an object_type if it has been merged with an object during deduplication,
// generate also the dynamic mapping for the object.
addObjectAsDynamicMapping(field);
mappings.hasDynamicTemplateMappings = true;
}
if (mappings.hasNonDynamicTemplateMappings) {
fieldProps = {
properties:
Expand Down

0 comments on commit e3877e0

Please sign in to comment.