Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[7.15] [Security Solution][Detections] Truncate lastFailureMessage for siem-detection-engine-rule-status documents (#112257) #115166

Merged
merged 1 commit into from
Oct 15, 2021
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

export * from './rule_execution_log_client';
export * from './types';
export * from './utils/normalization';
Original file line number Diff line number Diff line change
@@ -19,6 +19,7 @@ import {
LogStatusChangeArgs,
UpdateExecutionLogArgs,
} from './types';
import { truncateMessage } from './utils/normalization';

export interface RuleExecutionLogClientArgs {
ruleDataService: IRuleDataPluginService;
@@ -51,7 +52,16 @@ export class RuleExecutionLogClient implements IRuleExecutionLogClient {
}

public async update(args: UpdateExecutionLogArgs) {
return this.client.update(args);
const { lastFailureMessage, lastSuccessMessage, ...restAttributes } = args.attributes;

return this.client.update({
...args,
attributes: {
lastFailureMessage: truncateMessage(lastFailureMessage),
lastSuccessMessage: truncateMessage(lastSuccessMessage),
...restAttributes,
},
});
}

public async delete(id: string) {
@@ -63,6 +73,10 @@ export class RuleExecutionLogClient implements IRuleExecutionLogClient {
}

public async logStatusChange(args: LogStatusChangeArgs) {
return this.client.logStatusChange(args);
const message = args.message ? truncateMessage(args.message) : args.message;
return this.client.logStatusChange({
...args,
message,
});
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { take, toString, truncate, uniq } from 'lodash';

// When we write rule execution status updates to `siem-detection-engine-rule-status` saved objects
// or to event log, we write success and failure messages as well. Those messages are built from
// N errors collected during the "big loop" in the Detection Engine, where N can be very large.
// When N is large the resulting message strings are so large that these documents are up to 26MB.
// These large documents may cause migrations to fail because a batch of 1000 documents easily
// exceed Elasticsearch's `http.max_content_length` which defaults to 100mb.
// In order to fix that, we need to truncate those messages to an adequate MAX length.
// https://github.com/elastic/kibana/pull/112257

const MAX_MESSAGE_LENGTH = 10240;
const MAX_LIST_LENGTH = 20;

export const truncateMessage = (value: unknown): string | undefined => {
if (value === undefined) {
return value;
}

const str = toString(value);
return truncate(str, { length: MAX_MESSAGE_LENGTH });
};

export const truncateMessageList = (list: string[]): string[] => {
const deduplicatedList = uniq(list);
return take(deduplicatedList, MAX_LIST_LENGTH);
};
Original file line number Diff line number Diff line change
@@ -5,7 +5,8 @@
* 2.0.
*/

import { SavedObjectsType } from '../../../../../../../src/core/server';
import { SavedObjectsType, SavedObjectMigrationFn } from 'kibana/server';
import { truncateMessage } from '../rule_execution_log';

export const ruleStatusSavedObjectType = 'siem-detection-engine-rule-status';

@@ -47,11 +48,28 @@ export const ruleStatusSavedObjectMappings: SavedObjectsType['mappings'] = {
},
};

const truncateMessageFields: SavedObjectMigrationFn<Record<string, unknown>> = (doc) => {
const { lastFailureMessage, lastSuccessMessage, ...restAttributes } = doc.attributes;

return {
...doc,
attributes: {
lastFailureMessage: truncateMessage(lastFailureMessage),
lastSuccessMessage: truncateMessage(lastSuccessMessage),
...restAttributes,
},
references: doc.references ?? [],
};
};

export const type: SavedObjectsType = {
name: ruleStatusSavedObjectType,
hidden: false,
namespaceType: 'single',
mappings: ruleStatusSavedObjectMappings,
migrations: {
'7.15.2': truncateMessageFields,
},
};

export const ruleAssetSavedObjectType = 'security-rule';
Original file line number Diff line number Diff line change
@@ -70,8 +70,8 @@ import { wrapSequencesFactory } from './wrap_sequences_factory';
import { ConfigType } from '../../../config';
import { ExperimentalFeatures } from '../../../../common/experimental_features';
import { injectReferences, extractReferences } from './saved_object_references';
import { RuleExecutionLogClient } from '../rule_execution_log/rule_execution_log_client';
import { IRuleDataPluginService } from '../rule_execution_log/types';
import { RuleExecutionLogClient, truncateMessageList } from '../rule_execution_log';

export const signalRulesAlertType = ({
logger,
@@ -362,7 +362,9 @@ export const signalRulesAlertType = ({
throw new Error(`unknown rule type ${type}`);
}
if (result.warningMessages.length) {
const warningMessage = buildRuleMessage(result.warningMessages.join());
const warningMessage = buildRuleMessage(
truncateMessageList(result.warningMessages).join()
);
await ruleStatusService.partialFailure(warningMessage);
}

@@ -427,7 +429,7 @@ export const signalRulesAlertType = ({
} else {
const errorMessage = buildRuleMessage(
'Bulk Indexing of signals failed:',
result.errors.join()
truncateMessageList(result.errors).join()
);
logger.error(errorMessage);
await ruleStatusService.error(errorMessage, {