diff --git a/x-pack/legacy/plugins/ml/server/models/results_service/build_anomaly_table_items.d.ts b/x-pack/legacy/plugins/ml/server/models/results_service/build_anomaly_table_items.d.ts new file mode 100644 index 0000000000000..2bd19985c8518 --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/models/results_service/build_anomaly_table_items.d.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { AnomalyRecordDoc } from '../../../common/types/anomalies'; + +export interface AnomaliesTableRecord { + time: number; + source: AnomalyRecordDoc; + rowId: string; + jobId: string; + detectorIndex: number; + severity: number; + entityName?: string; + entityValue?: any; + influencers?: Array<{ [key: string]: any }>; + actual?: number[]; + actualSort?: any; + typical?: number[]; + typicalSort?: any; + metricDescriptionSort?: number; +} + +export function buildAnomalyTableItems( + anomalyRecords: any, + aggregationInterval: any, + dateFormatTz: string +): AnomaliesTableRecord[]; diff --git a/x-pack/legacy/plugins/ml/server/models/results_service/get_partition_fields_values.ts b/x-pack/legacy/plugins/ml/server/models/results_service/get_partition_fields_values.ts index 00e3002a7fc71..99eeaacc8de9c 100644 --- a/x-pack/legacy/plugins/ml/server/models/results_service/get_partition_fields_values.ts +++ b/x-pack/legacy/plugins/ml/server/models/results_service/get_partition_fields_values.ts @@ -7,11 +7,7 @@ import Boom from 'boom'; import { ML_RESULTS_INDEX_PATTERN } from '../../../common/constants/index_patterns'; import { callWithRequestType } from '../../../common/types/kibana'; - -interface CriteriaField { - fieldName: string; - fieldValue: any; -} +import { CriteriaField } from './results_service'; const PARTITION_FIELDS = ['partition_field', 'over_field', 'by_field'] as const; diff --git a/x-pack/legacy/plugins/ml/server/models/results_service/index.js b/x-pack/legacy/plugins/ml/server/models/results_service/index.ts similarity index 100% rename from x-pack/legacy/plugins/ml/server/models/results_service/index.js rename to x-pack/legacy/plugins/ml/server/models/results_service/index.ts diff --git a/x-pack/legacy/plugins/ml/server/models/results_service/results_service.js b/x-pack/legacy/plugins/ml/server/models/results_service/results_service.ts similarity index 81% rename from x-pack/legacy/plugins/ml/server/models/results_service/results_service.js rename to x-pack/legacy/plugins/ml/server/models/results_service/results_service.ts index 3ee5d04186ff1..5b154991f7cf0 100644 --- a/x-pack/legacy/plugins/ml/server/models/results_service/results_service.js +++ b/x-pack/legacy/plugins/ml/server/models/results_service/results_service.ts @@ -6,18 +6,33 @@ import _ from 'lodash'; import moment from 'moment'; - -import { buildAnomalyTableItems } from './build_anomaly_table_items'; +import { SearchResponse } from 'elasticsearch'; +import { RequestHandlerContext } from 'kibana/server'; +import { buildAnomalyTableItems, AnomaliesTableRecord } from './build_anomaly_table_items'; import { ML_RESULTS_INDEX_PATTERN } from '../../../common/constants/index_patterns'; import { ANOMALIES_TABLE_DEFAULT_QUERY_SIZE } from '../../../common/constants/search'; import { getPartitionFieldsValuesFactory } from './get_partition_fields_values'; +import { AnomalyRecordDoc } from '../../../common/types/anomalies'; // Service for carrying out Elasticsearch queries to obtain data for the // ML Results dashboards. const DEFAULT_MAX_EXAMPLES = 500; -export function resultsServiceProvider(callWithRequest) { +export interface CriteriaField { + fieldType?: string; + fieldName: string; + fieldValue: any; +} + +interface Influencer { + fieldName: string; + fieldValue: any; +} + +export function resultsServiceProvider(client: RequestHandlerContext | (() => any)) { + const callAsCurrentUser = + typeof client === 'object' ? client.ml!.mlClient.callAsCurrentUser : client; // Obtains data for the anomalies table, aggregating anomalies by day or hour as requested. // Return an Object with properties 'anomalies' and 'interval' (interval used to aggregate anomalies, // one of day, hour or second. Note 'auto' can be provided as the aggregationInterval in the request, @@ -25,21 +40,21 @@ export function resultsServiceProvider(callWithRequest) { // last anomalies), plus an examplesByJobId property if any of the // anomalies are categorization anomalies in mlcategory. async function getAnomaliesTableData( - jobIds, - criteriaFields, - influencers, - aggregationInterval, - threshold, - earliestMs, - latestMs, - dateFormatTz, - maxRecords = ANOMALIES_TABLE_DEFAULT_QUERY_SIZE, - maxExamples = DEFAULT_MAX_EXAMPLES, - influencersFilterQuery + jobIds: string[], + criteriaFields: CriteriaField[], + influencers: Influencer[], + aggregationInterval: string, + threshold: number, + earliestMs: number, + latestMs: number, + dateFormatTz: string, + maxRecords: number = ANOMALIES_TABLE_DEFAULT_QUERY_SIZE, + maxExamples: number = DEFAULT_MAX_EXAMPLES, + influencersFilterQuery: any ) { // Build the query to return the matching anomaly record results. // Add criteria for the time range, record score, plus any specified job IDs. - const boolCriteria = [ + const boolCriteria: object[] = [ { range: { timestamp: { @@ -120,7 +135,7 @@ export function resultsServiceProvider(callWithRequest) { }); } - const resp = await callWithRequest('search', { + const resp: SearchResponse = await callAsCurrentUser('search', { index: ML_RESULTS_INDEX_PATTERN, rest_total_hits_as_int: true, size: maxRecords, @@ -146,9 +161,16 @@ export function resultsServiceProvider(callWithRequest) { }, }); - const tableData = { anomalies: [], interval: 'second' }; + const tableData: { + anomalies: AnomaliesTableRecord[]; + interval: string; + examplesByJobId?: { [key: string]: any }; + } = { + anomalies: [], + interval: 'second', + }; if (resp.hits.total !== 0) { - let records = []; + let records: AnomalyRecordDoc[] = []; resp.hits.hits.forEach(hit => { records.push(hit._source); }); @@ -169,12 +191,12 @@ export function resultsServiceProvider(callWithRequest) { // Load examples for any categorization anomalies. const categoryAnomalies = tableData.anomalies.filter( - item => item.entityName === 'mlcategory' + (item: any) => item.entityName === 'mlcategory' ); if (categoryAnomalies.length > 0) { tableData.examplesByJobId = {}; - const categoryIdsByJobId = {}; + const categoryIdsByJobId: { [key: string]: any } = {}; categoryAnomalies.forEach(anomaly => { if (!_.has(categoryIdsByJobId, anomaly.jobId)) { categoryIdsByJobId[anomaly.jobId] = []; @@ -192,7 +214,9 @@ export function resultsServiceProvider(callWithRequest) { categoryIdsByJobId[jobId], maxExamples ); - tableData.examplesByJobId[jobId] = examplesByCategoryId; + if (tableData.examplesByJobId !== undefined) { + tableData.examplesByJobId[jobId] = examplesByCategoryId; + } }) ); } @@ -202,10 +226,10 @@ export function resultsServiceProvider(callWithRequest) { } // Returns the maximum anomaly_score for result_type:bucket over jobIds for the interval passed in - async function getMaxAnomalyScore(jobIds = [], earliestMs, latestMs) { + async function getMaxAnomalyScore(jobIds: string[] = [], earliestMs: number, latestMs: number) { // Build the criteria to use in the bool filter part of the request. // Adds criteria for the time range plus any specified job IDs. - const boolCriteria = [ + const boolCriteria: object[] = [ { range: { timestamp: { @@ -265,7 +289,7 @@ export function resultsServiceProvider(callWithRequest) { }, }; - const resp = await callWithRequest('search', query); + const resp = await callAsCurrentUser('search', query); const maxScore = _.get(resp, ['aggregations', 'max_score', 'value'], null); return { maxScore }; @@ -275,7 +299,7 @@ export function resultsServiceProvider(callWithRequest) { // Returns data over all jobs unless an optional list of job IDs of interest is supplied. // Returned response consists of latest bucket timestamps (ms since Jan 1 1970) against job ID async function getLatestBucketTimestampByJob(jobIds = []) { - const filter = [ + const filter: object[] = [ { term: { result_type: 'bucket', @@ -303,7 +327,7 @@ export function resultsServiceProvider(callWithRequest) { // Size of job terms agg, consistent with maximum number of jobs supported by Java endpoints. const maxJobs = 10000; - const resp = await callWithRequest('search', { + const resp = await callAsCurrentUser('search', { index: ML_RESULTS_INDEX_PATTERN, size: 0, body: { @@ -330,8 +354,12 @@ export function resultsServiceProvider(callWithRequest) { }, }); - const bucketsByJobId = _.get(resp, ['aggregations', 'byJobId', 'buckets'], []); - const timestampByJobId = {}; + const bucketsByJobId: Array<{ key: string; maxTimestamp: { value?: number } }> = _.get( + resp, + ['aggregations', 'byJobId', 'buckets'], + [] + ); + const timestampByJobId: { [key: string]: number | undefined } = {}; bucketsByJobId.forEach(bucket => { timestampByJobId[bucket.key] = bucket.maxTimestamp.value; }); @@ -342,8 +370,8 @@ export function resultsServiceProvider(callWithRequest) { // Obtains the categorization examples for the categories with the specified IDs // from the given index and job ID. // Returned response consists of a list of examples against category ID. - async function getCategoryExamples(jobId, categoryIds, maxExamples) { - const resp = await callWithRequest('search', { + async function getCategoryExamples(jobId: string, categoryIds: any, maxExamples: number) { + const resp = await callAsCurrentUser('search', { index: ML_RESULTS_INDEX_PATTERN, rest_total_hits_as_int: true, size: ANOMALIES_TABLE_DEFAULT_QUERY_SIZE, // Matches size of records in anomaly summary table. @@ -356,9 +384,9 @@ export function resultsServiceProvider(callWithRequest) { }, }); - const examplesByCategoryId = {}; + const examplesByCategoryId: { [key: string]: any } = {}; if (resp.hits.total !== 0) { - resp.hits.hits.forEach(hit => { + resp.hits.hits.forEach((hit: any) => { if (maxExamples) { examplesByCategoryId[hit._source.category_id] = _.slice( hit._source.examples, @@ -377,8 +405,8 @@ export function resultsServiceProvider(callWithRequest) { // Obtains the definition of the category with the specified ID and job ID. // Returned response contains four properties - categoryId, regex, examples // and terms (space delimited String of the common tokens matched in values of the category). - async function getCategoryDefinition(jobId, categoryId) { - const resp = await callWithRequest('search', { + async function getCategoryDefinition(jobId: string, categoryId: string) { + const resp = await callAsCurrentUser('search', { index: ML_RESULTS_INDEX_PATTERN, rest_total_hits_as_int: true, size: 1, @@ -409,6 +437,6 @@ export function resultsServiceProvider(callWithRequest) { getCategoryExamples, getLatestBucketTimestampByJob, getMaxAnomalyScore, - getPartitionFieldsValues: getPartitionFieldsValuesFactory(callWithRequest), + getPartitionFieldsValues: getPartitionFieldsValuesFactory(callAsCurrentUser), }; } diff --git a/x-pack/legacy/plugins/ml/server/new_platform/results_service_schema.ts b/x-pack/legacy/plugins/ml/server/new_platform/results_service_schema.ts new file mode 100644 index 0000000000000..b9a70b8e14197 --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/new_platform/results_service_schema.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { schema } from '@kbn/config-schema'; + +const criteriaFieldSchema = schema.object({ + fieldType: schema.maybe(schema.string()), + fieldName: schema.string(), + fieldValue: schema.any(), +}); + +export const anomaliesTableDataSchema = { + jobIds: schema.arrayOf(schema.string()), + criteriaFields: schema.arrayOf(criteriaFieldSchema), + influencers: schema.arrayOf(schema.maybe(schema.string())), + aggregationInterval: schema.string(), + threshold: schema.number(), + earliestMs: schema.number(), + latestMs: schema.number(), + dateFormatTz: schema.string(), + maxRecords: schema.number(), + maxExamples: schema.maybe(schema.number()), + influencersFilterQuery: schema.maybe(schema.any()), +}; + +export const partitionFieldValuesSchema = { + jobId: schema.string(), + searchTerm: schema.maybe(schema.any()), + criteriaFields: schema.arrayOf(criteriaFieldSchema), + earliestMs: schema.number(), + latestMs: schema.number(), +}; diff --git a/x-pack/legacy/plugins/ml/server/routes/apidoc.json b/x-pack/legacy/plugins/ml/server/routes/apidoc.json index f2ed202ae4777..574065446827d 100644 --- a/x-pack/legacy/plugins/ml/server/routes/apidoc.json +++ b/x-pack/legacy/plugins/ml/server/routes/apidoc.json @@ -33,6 +33,12 @@ "ValidateAnomalyDetector", "ForecastAnomalyDetector", "GetOverallBuckets", - "GetCategories" + "GetCategories", + "ResultsService", + "GetAnomaliesTableData", + "GetCategoryDefinition", + "GetMaxAnomalyScore", + "GetCategoryExamples", + "GetPartitionFieldsValues" ] } diff --git a/x-pack/legacy/plugins/ml/server/routes/results_service.js b/x-pack/legacy/plugins/ml/server/routes/results_service.js deleted file mode 100644 index a658729e85083..0000000000000 --- a/x-pack/legacy/plugins/ml/server/routes/results_service.js +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { callWithRequestFactory } from '../client/call_with_request_factory'; -import { wrapError } from '../client/errors'; -import { resultsServiceProvider } from '../models/results_service'; - -function getAnomaliesTableData(callWithRequest, payload) { - const rs = resultsServiceProvider(callWithRequest); - const { - jobIds, - criteriaFields, - influencers, - aggregationInterval, - threshold, - earliestMs, - latestMs, - dateFormatTz, - maxRecords, - maxExamples, - influencersFilterQuery, - } = payload; - return rs.getAnomaliesTableData( - jobIds, - criteriaFields, - influencers, - aggregationInterval, - threshold, - earliestMs, - latestMs, - dateFormatTz, - maxRecords, - maxExamples, - influencersFilterQuery - ); -} - -function getCategoryDefinition(callWithRequest, payload) { - const rs = resultsServiceProvider(callWithRequest); - return rs.getCategoryDefinition(payload.jobId, payload.categoryId); -} - -function getCategoryExamples(callWithRequest, payload) { - const rs = resultsServiceProvider(callWithRequest); - const { jobId, categoryIds, maxExamples } = payload; - return rs.getCategoryExamples(jobId, categoryIds, maxExamples); -} - -function getMaxAnomalyScore(callWithRequest, payload) { - const rs = resultsServiceProvider(callWithRequest); - const { jobIds, earliestMs, latestMs } = payload; - return rs.getMaxAnomalyScore(jobIds, earliestMs, latestMs); -} - -function getPartitionFieldsValues(callWithRequest, payload) { - const rs = resultsServiceProvider(callWithRequest); - const { jobId, searchTerm, criteriaFields, earliestMs, latestMs } = payload; - return rs.getPartitionFieldsValues(jobId, searchTerm, criteriaFields, earliestMs, latestMs); -} - -export function resultsServiceRoutes({ commonRouteConfig, elasticsearchPlugin, route }) { - route({ - method: 'POST', - path: '/api/ml/results/anomalies_table_data', - handler(request) { - const callWithRequest = callWithRequestFactory(elasticsearchPlugin, request); - return getAnomaliesTableData(callWithRequest, request.payload).catch(resp => wrapError(resp)); - }, - config: { - ...commonRouteConfig, - }, - }); - - route({ - method: 'POST', - path: '/api/ml/results/category_definition', - handler(request) { - const callWithRequest = callWithRequestFactory(elasticsearchPlugin, request); - return getCategoryDefinition(callWithRequest, request.payload).catch(resp => wrapError(resp)); - }, - config: { - ...commonRouteConfig, - }, - }); - - route({ - method: 'POST', - path: '/api/ml/results/max_anomaly_score', - handler(request) { - const callWithRequest = callWithRequestFactory(elasticsearchPlugin, request); - return getMaxAnomalyScore(callWithRequest, request.payload).catch(resp => wrapError(resp)); - }, - config: { - ...commonRouteConfig, - }, - }); - - route({ - method: 'POST', - path: '/api/ml/results/category_examples', - handler(request) { - const callWithRequest = callWithRequestFactory(elasticsearchPlugin, request); - return getCategoryExamples(callWithRequest, request.payload).catch(resp => wrapError(resp)); - }, - config: { - ...commonRouteConfig, - }, - }); - - route({ - method: 'POST', - path: '/api/ml/results/partition_fields_values', - handler(request) { - const callWithRequest = callWithRequestFactory(elasticsearchPlugin, request); - return getPartitionFieldsValues(callWithRequest, request.payload).catch(resp => - wrapError(resp) - ); - }, - config: { - ...commonRouteConfig, - }, - }); -} diff --git a/x-pack/legacy/plugins/ml/server/routes/results_service.ts b/x-pack/legacy/plugins/ml/server/routes/results_service.ts new file mode 100644 index 0000000000000..b44b82ec486d7 --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/routes/results_service.ts @@ -0,0 +1,220 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { RequestHandlerContext } from 'src/core/server'; +import { schema } from '@kbn/config-schema'; +import { licensePreRoutingFactory } from '../new_platform/licence_check_pre_routing_factory'; +import { wrapError } from '../client/error_wrapper'; +import { RouteInitialization } from '../new_platform/plugin'; +import { + anomaliesTableDataSchema, + partitionFieldValuesSchema, +} from '../new_platform/results_service_schema'; +import { resultsServiceProvider } from '../models/results_service'; + +function getAnomaliesTableData(context: RequestHandlerContext, payload: any) { + const rs = resultsServiceProvider(context); + const { + jobIds, + criteriaFields, + influencers, + aggregationInterval, + threshold, + earliestMs, + latestMs, + dateFormatTz, + maxRecords, + maxExamples, + influencersFilterQuery, + } = payload; + return rs.getAnomaliesTableData( + jobIds, + criteriaFields, + influencers, + aggregationInterval, + threshold, + earliestMs, + latestMs, + dateFormatTz, + maxRecords, + maxExamples, + influencersFilterQuery + ); +} + +function getCategoryDefinition(context: RequestHandlerContext, payload: any) { + const rs = resultsServiceProvider(context); + return rs.getCategoryDefinition(payload.jobId, payload.categoryId); +} + +function getCategoryExamples(context: RequestHandlerContext, payload: any) { + const rs = resultsServiceProvider(context); + const { jobId, categoryIds, maxExamples } = payload; + return rs.getCategoryExamples(jobId, categoryIds, maxExamples); +} + +function getMaxAnomalyScore(context: RequestHandlerContext, payload: any) { + const rs = resultsServiceProvider(context); + const { jobIds, earliestMs, latestMs } = payload; + return rs.getMaxAnomalyScore(jobIds, earliestMs, latestMs); +} + +function getPartitionFieldsValues(context: RequestHandlerContext, payload: any) { + const rs = resultsServiceProvider(context); + const { jobId, searchTerm, criteriaFields, earliestMs, latestMs } = payload; + return rs.getPartitionFieldsValues(jobId, searchTerm, criteriaFields, earliestMs, latestMs); +} + +/** + * Routes for results service + */ +export function resultsServiceRoutes({ xpackMainPlugin, router }: RouteInitialization) { + /** + * @apiGroup ResultsService + * + * @api {post} /api/ml/results/anomalies_table_data Prepare anomalies records for table display + * @apiName GetAnomaliesTableData + * @apiDescription Retrieves anomaly records for an anomaly detection job and formats them for anomalies table display + */ + router.post( + { + path: '/api/ml/results/anomalies_table_data', + validate: { + body: schema.object({ ...anomaliesTableDataSchema }), + }, + }, + licensePreRoutingFactory(xpackMainPlugin, async (context, request, response) => { + try { + const resp = await getAnomaliesTableData(context, request.body); + + return response.ok({ + body: resp, + }); + } catch (e) { + return response.customError(wrapError(e)); + } + }) + ); + + /** + * @apiGroup ResultsService + * + * @api {post} /api/ml/results/category_definition Returns category definition + * @apiName GetCategoryDefinition + * @apiDescription Returns the definition of the category with the specified ID and job ID + */ + router.post( + { + path: '/api/ml/results/category_definition', + validate: { + body: schema.object({ + jobId: schema.maybe(schema.string()), + categoryId: schema.string(), + }), + }, + }, + licensePreRoutingFactory(xpackMainPlugin, async (context, request, response) => { + try { + const resp = await getCategoryDefinition(context, request.body); + + return response.ok({ + body: resp, + }); + } catch (e) { + return response.customError(wrapError(e)); + } + }) + ); + + /** + * @apiGroup ResultsService + * + * @api {post} /api/ml/results/max_anomaly_score Returns the maximum anomaly_score + * @apiName GetMaxAnomalyScore + * @apiDescription Returns the maximum anomaly score of the bucket results for the request job ID(s) and time range + */ + router.post( + { + path: '/api/ml/results/max_anomaly_score', + validate: { + body: schema.object({ + jobIds: schema.arrayOf(schema.string()), + earliestMs: schema.number(), + latestMs: schema.number(), + }), + }, + }, + licensePreRoutingFactory(xpackMainPlugin, async (context, request, response) => { + try { + const resp = await getMaxAnomalyScore(context, request.body); + + return response.ok({ + body: resp, + }); + } catch (e) { + return response.customError(wrapError(e)); + } + }) + ); + + /** + * @apiGroup ResultsService + * + * @api {post} /api/ml/results/category_examples Returns category examples + * @apiName GetCategoryExamples + * @apiDescription Returns examples for the categories with the specified IDs from the job with the supplied ID + */ + router.post( + { + path: '/api/ml/results/category_examples', + validate: { + body: schema.object({ + jobId: schema.string(), + categoryIds: schema.arrayOf(schema.string()), + maxExamples: schema.number(), + }), + }, + }, + licensePreRoutingFactory(xpackMainPlugin, async (context, request, response) => { + try { + const resp = await getCategoryExamples(context, request.body); + + return response.ok({ + body: resp, + }); + } catch (e) { + return response.customError(wrapError(e)); + } + }) + ); + + /** + * @apiGroup ResultsService + * + * @api {post} /api/ml/results/partition_fields_values Returns partition fields values + * @apiName GetPartitionFieldsValues + * @apiDescription Returns the partition fields with values that match the provided criteria for the specified job ID. + */ + router.post( + { + path: '/api/ml/results/partition_fields_values', + validate: { + body: schema.object({ ...partitionFieldValuesSchema }), + }, + }, + licensePreRoutingFactory(xpackMainPlugin, async (context, request, response) => { + try { + const resp = await getPartitionFieldsValues(context, request.body); + + return response.ok({ + body: resp, + }); + } catch (e) { + return response.customError(wrapError(e)); + } + }) + ); +}