Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Logs UI] Add expandable rows with category examples #54586

Merged
Merged
Changes from 1 commit
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
298d547
Add category examples route
weltenwort Jan 9, 2020
59b37bc
Show examples in category table row expansions
weltenwort Jan 10, 2020
7d58cd2
Use `transport.request` fallback to query ml jobs
weltenwort Feb 3, 2020
fc75abf
Prepare log rows and columns for re-use
weltenwort Feb 4, 2020
a751fd9
Render example message using log stream components
weltenwort Feb 4, 2020
ba2b552
Introduce a stricter wrapping mode for the examples
weltenwort Feb 4, 2020
551df33
Add example loading indicators
weltenwort Feb 4, 2020
2645faf
Add example loading failure message with retry option
weltenwort Feb 4, 2020
eaceb39
Remove mismatching log message background color
weltenwort Feb 5, 2020
687bdfb
Merge branch 'master' into logs-ui-categorization-add-examples
weltenwort Feb 6, 2020
75e4f4c
Add dataset column to examples and align columns
weltenwort Feb 11, 2020
e82cc54
Merge branch 'master' into logs-ui-categorization-add-examples
weltenwort Feb 11, 2020
b668695
Merge branch 'master' into logs-ui-categorization-add-examples
weltenwort Feb 11, 2020
704e123
Remove unused translations via autofix
weltenwort Feb 11, 2020
3074c48
Merge branch 'master' into logs-ui-categorization-add-examples
weltenwort Feb 28, 2020
40bbbac
Merge branch 'master' into logs-ui-categorization-add-examples
weltenwort Mar 2, 2020
422dfea
Display per-dataset anomaly scores and ML links
weltenwort Mar 3, 2020
182098d
Adjust dataset column width in example rows
weltenwort Mar 3, 2020
55db708
Add message for empty example response
weltenwort Mar 3, 2020
ff6521e
Merge branch 'master' into logs-ui-categorization-add-examples
weltenwort Mar 3, 2020
6e23015
Merge branch 'master' into logs-ui-categorization-add-examples
elasticmachine Mar 5, 2020
38c6e42
Merge branch 'master' into logs-ui-categorization-add-examples
elasticmachine Mar 5, 2020
0544202
Merge branch 'master' into logs-ui-categorization-add-examples
weltenwort Mar 9, 2020
3e818cd
Merge branch 'master' into logs-ui-categorization-add-examples
weltenwort Mar 11, 2020
aea5ece
Use new link helper hook for dataset ml link
weltenwort Mar 11, 2020
e283dc8
Correctly read dataset from sample message documents
weltenwort Mar 11, 2020
1b0e3ee
Rename 'original' to 'pre-wrapped' wrap mode
weltenwort Mar 11, 2020
f52e407
Merge branch 'master' into logs-ui-categorization-add-examples
elasticmachine Mar 11, 2020
90cbc6f
Merge branch 'master' into logs-ui-categorization-add-examples
elasticmachine Mar 11, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Add category examples route
weltenwort committed Feb 3, 2020
commit 298d54781738071b6faa287549db9f6ed8c0d248
Original file line number Diff line number Diff line change
@@ -6,4 +6,5 @@

export * from './log_entry_categories';
export * from './log_entry_category_datasets';
export * from './log_entry_category_examples';
export * from './log_entry_rate';
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import * as rt from 'io-ts';

import {
badRequestErrorRT,
forbiddenErrorRT,
timeRangeRT,
routeTimingMetadataRT,
} from '../../shared';

export const LOG_ANALYSIS_GET_LOG_ENTRY_CATEGORY_EXAMPLES_PATH =
'/api/infra/log_analysis/results/log_entry_category_examples';

/**
* request
*/

export const getLogEntryCategoryExamplesRequestPayloadRT = rt.type({
data: rt.type({
// the category to fetch the examples for
categoryId: rt.number,
// the number of examples to fetch
exampleCount: rt.number,
// the id of the source configuration
sourceId: rt.string,
// the time range to fetch the category examples from
timeRange: timeRangeRT,
}),
});

export type GetLogEntryCategoryExamplesRequestPayload = rt.TypeOf<
typeof getLogEntryCategoryExamplesRequestPayloadRT
>;

/**
* response
*/

const logEntryCategoryExampleRT = rt.type({
timestamp: rt.number,
message: rt.string,
});

export type LogEntryCategoryExample = rt.TypeOf<typeof logEntryCategoryExampleRT>;

export const getLogEntryCategoryExamplesSuccessReponsePayloadRT = rt.intersection([
rt.type({
data: rt.type({
examples: rt.array(logEntryCategoryExampleRT),
}),
}),
rt.partial({
timing: routeTimingMetadataRT,
}),
]);

export type GetLogEntryCategoryExamplesSuccessResponsePayload = rt.TypeOf<
typeof getLogEntryCategoryExamplesSuccessReponsePayloadRT
>;

export const getLogEntryCategoryExamplesResponsePayloadRT = rt.union([
getLogEntryCategoryExamplesSuccessReponsePayloadRT,
badRequestErrorRT,
forbiddenErrorRT,
]);

export type GetLogEntryCategoryExamplesReponsePayload = rt.TypeOf<
typeof getLogEntryCategoryExamplesResponsePayloadRT
>;
Original file line number Diff line number Diff line change
@@ -28,3 +28,10 @@ export const jobSourceConfigurationRT = rt.type({
});

export type JobSourceConfiguration = rt.TypeOf<typeof jobSourceConfigurationRT>;

export const jobCustomSettingsRT = rt.partial({
job_revision: rt.number,
logs_source_config: rt.partial(jobSourceConfigurationRT.props),
});

export type JobCustomSettings = rt.TypeOf<typeof jobCustomSettingsRT>;
Original file line number Diff line number Diff line change
@@ -6,13 +6,6 @@

import * as rt from 'io-ts';

import { jobSourceConfigurationRT } from '../../../../../common/log_analysis';

export const jobCustomSettingsRT = rt.partial({
job_revision: rt.number,
logs_source_config: rt.partial(jobSourceConfigurationRT.props),
});

export const getMlCapabilitiesResponsePayloadRT = rt.type({
capabilities: rt.type({
canGetJobs: rt.boolean,
Original file line number Diff line number Diff line change
@@ -4,14 +4,14 @@
* you may not use this file except in compliance with the Elastic License.
*/

import { pipe } from 'fp-ts/lib/pipeable';
import { fold } from 'fp-ts/lib/Either';
import { identity } from 'fp-ts/lib/function';
import { pipe } from 'fp-ts/lib/pipeable';
import * as rt from 'io-ts';
import { npStart } from 'ui/new_platform';
import { jobCustomSettingsRT } from './ml_api_types';
import { throwErrors, createPlainError } from '../../../../../common/runtime_types';
import { getJobId } from '../../../../../common/log_analysis';

import { getJobId, jobCustomSettingsRT } from '../../../../../common/log_analysis';
import { createPlainError, throwErrors } from '../../../../../common/runtime_types';

export const callJobsSummaryAPI = async <JobType extends string>(
spaceId: string,
Original file line number Diff line number Diff line change
@@ -5,12 +5,13 @@
*/

import { fold } from 'fp-ts/lib/Either';
import { pipe } from 'fp-ts/lib/pipeable';
import { identity } from 'fp-ts/lib/function';
import { pipe } from 'fp-ts/lib/pipeable';
import * as rt from 'io-ts';
import { npStart } from 'ui/new_platform';
import { throwErrors, createPlainError } from '../../../../../common/runtime_types';
import { jobCustomSettingsRT } from './ml_api_types';

import { jobCustomSettingsRT } from '../../../../../common/log_analysis';
import { createPlainError, throwErrors } from '../../../../../common/runtime_types';

export const callGetMlModuleAPI = async (moduleId: string) => {
const response = await npStart.core.http.fetch(`/api/ml/modules/get_module/${moduleId}`, {
Original file line number Diff line number Diff line change
@@ -5,12 +5,13 @@
*/

import { fold } from 'fp-ts/lib/Either';
import { pipe } from 'fp-ts/lib/pipeable';
import { identity } from 'fp-ts/lib/function';
import { pipe } from 'fp-ts/lib/pipeable';
import * as rt from 'io-ts';
import { npStart } from 'ui/new_platform';
import { throwErrors, createPlainError } from '../../../../../common/runtime_types';
import { getJobIdPrefix } from '../../../../../common/log_analysis';

import { getJobIdPrefix, jobCustomSettingsRT } from '../../../../../common/log_analysis';
import { createPlainError, throwErrors } from '../../../../../common/runtime_types';

export const callSetupMlModuleAPI = async (
moduleId: string,
@@ -48,7 +49,10 @@ const setupMlModuleTimeParamsRT = rt.partial({
end: rt.number,
});

const setupMlModuleJobOverridesRT = rt.object;
const setupMlModuleJobOverridesRT = rt.type({
job_id: rt.string,
custom_settings: jobCustomSettingsRT,
});

export type SetupMlModuleJobOverrides = rt.TypeOf<typeof setupMlModuleJobOverridesRT>;

2 changes: 2 additions & 0 deletions x-pack/legacy/plugins/infra/server/infra_server.ts
Original file line number Diff line number Diff line change
@@ -14,6 +14,7 @@ import { InfraBackendLibs } from './lib/infra_types';
import {
initGetLogEntryCategoriesRoute,
initGetLogEntryCategoryDatasetsRoute,
initGetLogEntryCategoryExamplesRoute,
initGetLogEntryRateRoute,
initValidateLogAnalysisIndicesRoute,
} from './routes/log_analysis';
@@ -45,6 +46,7 @@ export const initInfraServer = (libs: InfraBackendLibs) => {
initIpToHostName(libs);
initGetLogEntryCategoriesRoute(libs);
initGetLogEntryCategoryDatasetsRoute(libs);
initGetLogEntryCategoryExamplesRoute(libs);
initGetLogEntryRateRoute(libs);
initSnapshotRoute(libs);
initNodeDetailsRoute(libs);
Original file line number Diff line number Diff line change
@@ -176,6 +176,11 @@ export class KibanaFramework {
method: 'indices.get' | 'ml.getBuckets',
options?: object
): Promise<InfraDatabaseGetIndicesResponse>;
callWithRequest(
requestContext: RequestHandlerContext,
method: 'ml.getJobs',
options?: object
): Promise<object>;
callWithRequest(
requestContext: RequestHandlerContext,
endpoint: string,
23 changes: 23 additions & 0 deletions x-pack/legacy/plugins/infra/server/lib/log_analysis/errors.ts
Original file line number Diff line number Diff line change
@@ -4,9 +4,32 @@
* you may not use this file except in compliance with the Elastic License.
*/

/* eslint-disable max-classes-per-file */

export class NoLogAnalysisResultsIndexError extends Error {
constructor(message?: string) {
super(message);
Object.setPrototypeOf(this, new.target.prototype);
}
}

export class NoLogAnalysisMlJobError extends Error {
constructor(message?: string) {
super(message);
Object.setPrototypeOf(this, new.target.prototype);
}
}

export class InsufficientLogAnalysisMlJobConfigurationError extends Error {
constructor(message?: string) {
super(message);
Object.setPrototypeOf(this, new.target.prototype);
}
}

export class UnknownCategoryError extends Error {
constructor(categoryId: number) {
super(`Unknown ml category ${categoryId}`);
Object.setPrototypeOf(this, new.target.prototype);
}
}
Original file line number Diff line number Diff line change
@@ -5,11 +5,20 @@
*/

import { KibanaRequest, RequestHandlerContext } from '../../../../../../../src/core/server';
import { getJobId, logEntryCategoriesJobTypes } from '../../../common/log_analysis';
import {
getJobId,
logEntryCategoriesJobTypes,
jobCustomSettingsRT,
} from '../../../common/log_analysis';
import { startTracingSpan, TracingSpan } from '../../../common/performance_tracing';
import { decodeOrThrow } from '../../../common/runtime_types';
import { KibanaFramework } from '../adapters/framework/kibana_framework_adapter';
import { NoLogAnalysisResultsIndexError } from './errors';
import {
NoLogAnalysisResultsIndexError,
NoLogAnalysisMlJobError,
InsufficientLogAnalysisMlJobConfigurationError,
UnknownCategoryError,
} from './errors';
import {
createLogEntryCategoriesQuery,
logEntryCategoriesResponseRT,
@@ -25,10 +34,15 @@ import {
LogEntryDatasetBucket,
logEntryDatasetsResponseRT,
} from './queries/log_entry_data_sets';
import { mlJobsResponseRT, createMlJobsQuery } from './queries/ml_jobs';
import {
createTopLogEntryCategoriesQuery,
topLogEntryCategoriesResponseRT,
} from './queries/top_log_entry_categories';
import {
createLogEntryCategoryExamplesQuery,
logEntryCategoryExamplesResponseRT,
} from './queries/log_entry_category_examples';

const COMPOSITE_AGGREGATION_BATCH_SIZE = 1000;

@@ -175,6 +189,80 @@ export class LogEntryCategoriesAnalysis {
};
}

public async getLogEntryCategoryExamples(
requestContext: RequestHandlerContext,
request: KibanaRequest,
sourceId: string,
startTime: number,
endTime: number,
categoryId: number,
exampleCount: number
) {
const finalizeLogEntryCategoryExamplesSpan = startTracingSpan(
'get category example log entries'
);

const logEntryCategoriesCountJobId = getJobId(
this.libs.framework.getSpaceId(request),
sourceId,
logEntryCategoriesJobTypes[0]
);

const {
mlJob,
timing: { spans: fetchMlJobSpans },
} = await this.fetchMlJob(requestContext, logEntryCategoriesCountJobId);

const customSettings = decodeOrThrow(jobCustomSettingsRT)(mlJob.custom_settings);
const indices = customSettings?.logs_source_config?.indexPattern;
const timestampField = customSettings?.logs_source_config?.timestampField;

if (indices == null || timestampField == null) {
throw new InsufficientLogAnalysisMlJobConfigurationError(
`Failed to find index configuration for ml job ${logEntryCategoriesCountJobId}`
);
}

const {
logEntryCategoriesById,
timing: { spans: fetchLogEntryCategoriesSpans },
} = await this.fetchLogEntryCategories(requestContext, logEntryCategoriesCountJobId, [
categoryId,
]);
const category = logEntryCategoriesById[categoryId];

if (category == null) {
throw new UnknownCategoryError(categoryId);
}

const {
examples,
timing: { spans: fetchLogEntryCategoryExamplesSpans },
} = await this.fetchLogEntryCategoryExamples(
requestContext,
indices,
timestampField,
startTime,
endTime,
category._source.terms,
exampleCount
);

const logEntryCategoryExamplesSpan = finalizeLogEntryCategoryExamplesSpan();

return {
data: examples,
timing: {
spans: [
logEntryCategoryExamplesSpan,
...fetchMlJobSpans,
...fetchLogEntryCategoriesSpans,
...fetchLogEntryCategoryExamplesSpans,
],
},
};
}

private async fetchTopLogEntryCategories(
requestContext: RequestHandlerContext,
logEntryCategoriesCountJobId: string,
@@ -351,6 +439,77 @@ export class LogEntryCategoriesAnalysis {
},
};
}

private async fetchMlJob(
requestContext: RequestHandlerContext,
logEntryCategoriesCountJobId: string
) {
const finalizeMlGetJobSpan = startTracingSpan('Fetch ml job from ES');

const {
jobs: [mlJob],
} = decodeOrThrow(mlJobsResponseRT)(
await this.libs.framework.callWithRequest(
requestContext,
'ml.getJobs',
createMlJobsQuery([logEntryCategoriesCountJobId])
)
);

const mlGetJobSpan = finalizeMlGetJobSpan();

if (mlJob == null) {
throw new NoLogAnalysisMlJobError(`Failed to find ml job ${logEntryCategoriesCountJobId}.`);
}

return {
mlJob,
timing: {
spans: [mlGetJobSpan],
},
};
}

private async fetchLogEntryCategoryExamples(
requestContext: RequestHandlerContext,
indices: string,
timestampField: string,
startTime: number,
endTime: number,
categoryQuery: string,
exampleCount: number
) {
const finalizeEsSearchSpan = startTracingSpan('Fetch examples from ES');

const {
hits: { hits },
} = decodeOrThrow(logEntryCategoryExamplesResponseRT)(
await this.libs.framework.callWithRequest(
requestContext,
'search',
createLogEntryCategoryExamplesQuery(
indices,
timestampField,
startTime,
endTime,
categoryQuery,
exampleCount
)
)
);

const esSearchSpan = finalizeEsSearchSpan();

return {
examples: hits.map(hit => ({
timestamp: hit.sort[0],
message: hit._source.message,
})),
timing: {
spans: [esSearchSpan],
},
};
}
}

const parseCategoryId = (rawCategoryId: string) => parseInt(rawCategoryId, 10);
Original file line number Diff line number Diff line change
@@ -35,3 +35,11 @@ export const createResultTypeFilters = (resultType: 'model_plot' | 'record') =>
},
},
];

export const createCategoryIdFilters = (categoryIds: number[]) => [
{
terms: {
category_id: categoryIds,
},
},
];
Original file line number Diff line number Diff line change
@@ -7,7 +7,7 @@
import * as rt from 'io-ts';

import { commonSearchSuccessResponseFieldsRT } from '../../../utils/elasticsearch_runtime_types';
import { defaultRequestParameters, getMlResultIndex } from './common';
import { defaultRequestParameters, getMlResultIndex, createCategoryIdFilters } from './common';

export const createLogEntryCategoriesQuery = (
logEntryCategoriesJobId: string,
@@ -17,16 +17,10 @@ export const createLogEntryCategoriesQuery = (
body: {
query: {
bool: {
filter: [
{
terms: {
category_id: categoryIds,
},
},
],
filter: [...createCategoryIdFilters(categoryIds)],
},
},
_source: ['category_id', 'regex'],
_source: ['category_id', 'regex', 'terms'],
},
index: getMlResultIndex(logEntryCategoriesJobId),
size: categoryIds.length,
@@ -36,6 +30,7 @@ export const logEntryCategoryHitRT = rt.type({
_source: rt.type({
category_id: rt.number,
regex: rt.string,
terms: rt.string,
}),
});

Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import * as rt from 'io-ts';

import { commonSearchSuccessResponseFieldsRT } from '../../../utils/elasticsearch_runtime_types';
import { defaultRequestParameters } from './common';

export const createLogEntryCategoryExamplesQuery = (
indices: string,
timestampField: string,
startTime: number,
endTime: number,
categoryQuery: string,
exampleCount: number
) => ({
...defaultRequestParameters,
body: {
query: {
bool: {
filter: [
{
range: {
[timestampField]: {
gte: startTime,
lte: endTime,
},
},
},
{
match: {
message: {
query: categoryQuery,
operator: 'AND',
},
},
},
],
},
},
_source: ['message'],
},
index: indices,
size: exampleCount,
sort: [
{
[timestampField]: {
order: 'asc',
},
},
],
});

export const logEntryCategoryExampleHitRT = rt.type({
_source: rt.type({
message: rt.string,
}),
sort: rt.tuple([rt.number]),
});

export type LogEntryCategoryExampleHit = rt.TypeOf<typeof logEntryCategoryExampleHitRT>;

export const logEntryCategoryExamplesResponseRT = rt.intersection([
commonSearchSuccessResponseFieldsRT,
rt.type({
hits: rt.type({
hits: rt.array(logEntryCategoryExampleHitRT),
}),
}),
]);

export type logEntryCategoryExamplesResponse = rt.TypeOf<typeof logEntryCategoryExamplesResponseRT>;
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import * as rt from 'io-ts';

export const createMlJobsQuery = (jobIds: string[]) => ({
job_id: jobIds,
allow_no_jobs: true,
});

export const mlJobRT = rt.type({
job_id: rt.string,
custom_settings: rt.unknown,
});

export const mlJobsResponseRT = rt.type({
jobs: rt.array(mlJobRT),
});
Original file line number Diff line number Diff line change
@@ -6,4 +6,5 @@

export * from './log_entry_categories';
export * from './log_entry_category_datasets';
export * from './log_entry_category_examples';
export * from './log_entry_rate';
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import { schema } from '@kbn/config-schema';
import Boom from 'boom';
import { fold } from 'fp-ts/lib/Either';
import { identity } from 'fp-ts/lib/function';
import { pipe } from 'fp-ts/lib/pipeable';

import {
getLogEntryCategoryExamplesRequestPayloadRT,
getLogEntryCategoryExamplesSuccessReponsePayloadRT,
LOG_ANALYSIS_GET_LOG_ENTRY_CATEGORY_EXAMPLES_PATH,
} from '../../../../common/http_api/log_analysis';
import { throwErrors } from '../../../../common/runtime_types';
import { InfraBackendLibs } from '../../../lib/infra_types';
import { NoLogAnalysisResultsIndexError } from '../../../lib/log_analysis';

const anyObject = schema.object({}, { allowUnknowns: true });

export const initGetLogEntryCategoryExamplesRoute = ({
framework,
logEntryCategoriesAnalysis,
}: InfraBackendLibs) => {
framework.registerRoute(
{
method: 'post',
path: LOG_ANALYSIS_GET_LOG_ENTRY_CATEGORY_EXAMPLES_PATH,
validate: {
// short-circuit forced @kbn/config-schema validation so we can do io-ts validation
body: anyObject,
},
},
async (requestContext, request, response) => {
const {
data: {
categoryId,
exampleCount,
sourceId,
timeRange: { startTime, endTime },
},
} = pipe(
getLogEntryCategoryExamplesRequestPayloadRT.decode(request.body),
fold(throwErrors(Boom.badRequest), identity)
);

try {
const {
data: logEntryCategoryExamples,
timing,
} = await logEntryCategoriesAnalysis.getLogEntryCategoryExamples(
requestContext,
request,
sourceId,
startTime,
endTime,
categoryId,
exampleCount
);

return response.ok({
body: getLogEntryCategoryExamplesSuccessReponsePayloadRT.encode({
data: {
examples: logEntryCategoryExamples,
},
timing,
}),
});
} catch (e) {
const { statusCode = 500, message = 'Unknown error occurred' } = e;

if (e instanceof NoLogAnalysisResultsIndexError) {
return response.notFound({ body: { message } });
}

return response.customError({
statusCode,
body: { message },
});
}
}
);
};