From cf03c92eb8e2b798971a4b92ba9ffd9cda896bb4 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 16 Mar 2021 18:08:33 -0400 Subject: [PATCH] [ML] Use indices options in anomaly detection job wizards (#91830) (#94755) * [ML] WIP Datafeed preview refactor * adding indices options to ad job creation searches * update datafeed preview schema * updating types * recalculating wizard time range on JSON edit save * updating endpoint docs * fixing types * more type fixes * fixing missing runtime fields * using isPopulatedObject * adding indices options schema * fixing test * fixing schema Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: James Gowdy --- .../types/anomaly_detection_jobs/datafeed.ts | 2 +- x-pack/plugins/ml/common/types/job_service.ts | 19 ++- .../chart_loader.ts | 3 +- .../common/chart_loader/chart_loader.ts | 25 ++- .../new_job/common/chart_loader/searches.ts | 5 +- .../job_creator/advanced_job_creator.ts | 17 +- .../categorization_examples_loader.ts | 3 +- .../common/results_loader/results_loader.ts | 3 +- .../json_editor_flyout/json_editor_flyout.tsx | 18 +- .../estimate_bucket_span.ts | 4 +- .../metric_selection_summary.tsx | 3 +- .../multi_metric_view/metric_selection.tsx | 9 +- .../metric_selection_summary.tsx | 6 +- .../population_view/metric_selection.tsx | 9 +- .../metric_selection_summary.tsx | 9 +- .../single_metric_view/metric_selection.tsx | 3 +- .../metric_selection_summary.tsx | 3 +- .../components/time_range_step/time_range.tsx | 3 +- .../jobs/new_job/pages/new_job/page.tsx | 21 +-- .../application/services/job_service.js | 161 +----------------- .../services/ml_api_service/index.ts | 22 +-- .../services/ml_api_service/jobs.ts | 25 ++- .../results_service/results_service.d.ts | 4 +- .../results_service/results_service.js | 11 +- .../bucket_span_estimator.d.ts | 16 +- .../bucket_span_estimator.js | 32 +++- .../bucket_span_estimator.test.ts | 4 +- .../polled_data_checker.js | 4 +- .../single_series_checker.js | 14 +- .../models/fields_service/fields_service.ts | 8 +- .../ml/server/models/job_service/datafeeds.ts | 157 ++++++++++++++++- .../ml/server/models/job_service/index.ts | 4 +- .../ml/server/models/job_service/jobs.ts | 1 + .../models/job_service/model_snapshots.ts | 5 +- .../new_job/categorization/examples.ts | 11 +- .../models/job_service/new_job/line_chart.ts | 11 +- .../job_service/new_job/population_chart.ts | 11 +- x-pack/plugins/ml/server/routes/apidoc.json | 1 + .../apidoc_scripts/schema_extractor.test.ts | 2 +- .../ml/server/routes/fields_service.ts | 4 +- .../plugins/ml/server/routes/job_service.ts | 60 ++++++- .../server/routes/schemas/datafeeds_schema.ts | 26 ++- .../routes/schemas/fields_service_schema.ts | 3 + .../routes/schemas/job_service_schema.ts | 14 ++ .../routes/schemas/job_validation_schema.ts | 3 +- 45 files changed, 490 insertions(+), 289 deletions(-) diff --git a/x-pack/plugins/ml/common/types/anomaly_detection_jobs/datafeed.ts b/x-pack/plugins/ml/common/types/anomaly_detection_jobs/datafeed.ts index 06938485649fb..ed9c9e7589749 100644 --- a/x-pack/plugins/ml/common/types/anomaly_detection_jobs/datafeed.ts +++ b/x-pack/plugins/ml/common/types/anomaly_detection_jobs/datafeed.ts @@ -45,7 +45,7 @@ export type Aggregation = Record< } >; -interface IndicesOptions { +export interface IndicesOptions { expand_wildcards?: 'all' | 'open' | 'closed' | 'hidden' | 'none'; ignore_unavailable?: boolean; allow_no_indices?: boolean; diff --git a/x-pack/plugins/ml/common/types/job_service.ts b/x-pack/plugins/ml/common/types/job_service.ts index aae0b9c9b209f..5209743f87b3c 100644 --- a/x-pack/plugins/ml/common/types/job_service.ts +++ b/x-pack/plugins/ml/common/types/job_service.ts @@ -5,7 +5,9 @@ * 2.0. */ -import { Job, JobStats } from './anomaly_detection_jobs'; +import { Job, JobStats, IndicesOptions } from './anomaly_detection_jobs'; +import { RuntimeMappings } from './fields'; +import { ES_AGGREGATION } from '../constants/aggregation_types'; export interface MlJobsResponse { jobs: Job[]; @@ -23,3 +25,18 @@ export interface JobsExistResponse { isGroup: boolean; }; } + +export interface BucketSpanEstimatorData { + aggTypes: Array; + duration: { + start: number; + end: number; + }; + fields: Array; + index: string; + query: any; + splitField: string | undefined; + timeField: string | undefined; + runtimeMappings: RuntimeMappings | undefined; + indicesOptions: IndicesOptions | undefined; +} diff --git a/x-pack/plugins/ml/public/application/components/model_snapshots/revert_model_snapshot_flyout/chart_loader.ts b/x-pack/plugins/ml/public/application/components/model_snapshots/revert_model_snapshot_flyout/chart_loader.ts index 020f0d015eafe..a3753c8f000ae 100644 --- a/x-pack/plugins/ml/public/application/components/model_snapshots/revert_model_snapshot_flyout/chart_loader.ts +++ b/x-pack/plugins/ml/public/application/components/model_snapshots/revert_model_snapshot_flyout/chart_loader.ts @@ -29,7 +29,8 @@ export function chartLoaderProvider(mlResultsService: MlResultsService) { job.data_description.time_field, job.data_counts.earliest_record_timestamp, job.data_counts.latest_record_timestamp, - intervalMs + intervalMs, + job.datafeed_config.indices_options ); if (resp.error !== undefined) { throw resp.error; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/chart_loader/chart_loader.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/chart_loader/chart_loader.ts index a36e52f4e863b..ddd2aa3619472 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/chart_loader/chart_loader.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/chart_loader/chart_loader.ts @@ -8,6 +8,7 @@ import memoizeOne from 'memoize-one'; import { isEqual } from 'lodash'; import { IndexPatternTitle } from '../../../../../../common/types/kibana'; +import { IndicesOptions } from '../../../../../../common/types/anomaly_detection_jobs'; import { Field, SplitField, @@ -56,7 +57,8 @@ export class ChartLoader { splitField: SplitField, splitFieldValue: SplitFieldValue, intervalMs: number, - runtimeMappings: RuntimeMappings | null + runtimeMappings: RuntimeMappings | null, + indicesOptions?: IndicesOptions ): Promise { if (this._timeFieldName !== '') { if (aggFieldPairsCanBeCharted(aggFieldPairs) === false) { @@ -77,7 +79,8 @@ export class ChartLoader { aggFieldPairNames, splitFieldName, splitFieldValue, - runtimeMappings ?? undefined + runtimeMappings ?? undefined, + indicesOptions ); return resp.results; @@ -91,7 +94,8 @@ export class ChartLoader { aggFieldPairs: AggFieldPair[], splitField: SplitField, intervalMs: number, - runtimeMappings: RuntimeMappings | null + runtimeMappings: RuntimeMappings | null, + indicesOptions?: IndicesOptions ): Promise { if (this._timeFieldName !== '') { if (aggFieldPairsCanBeCharted(aggFieldPairs) === false) { @@ -111,7 +115,8 @@ export class ChartLoader { this._query, aggFieldPairNames, splitFieldName, - runtimeMappings ?? undefined + runtimeMappings ?? undefined, + indicesOptions ); return resp.results; @@ -122,7 +127,8 @@ export class ChartLoader { async loadEventRateChart( start: number, end: number, - intervalMs: number + intervalMs: number, + indicesOptions?: IndicesOptions ): Promise { if (this._timeFieldName !== '') { const resp = await getEventRateData( @@ -131,7 +137,8 @@ export class ChartLoader { this._timeFieldName, start, end, - intervalMs * 3 + intervalMs * 3, + indicesOptions ); if (resp.error !== undefined) { throw resp.error; @@ -147,14 +154,16 @@ export class ChartLoader { async loadFieldExampleValues( field: Field, - runtimeMappings: RuntimeMappings | null + runtimeMappings: RuntimeMappings | null, + indicesOptions?: IndicesOptions ): Promise { const { results } = await getCategoryFields( this._indexPatternTitle, field.name, 10, this._query, - runtimeMappings ?? undefined + runtimeMappings ?? undefined, + indicesOptions ); return results; } diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/chart_loader/searches.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/chart_loader/searches.ts index 54917c4884f22..499e1639f1fde 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/chart_loader/searches.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/chart_loader/searches.ts @@ -9,6 +9,7 @@ import { get } from 'lodash'; import { ml } from '../../../../services/ml_api_service'; import { RuntimeMappings } from '../../../../../../common/types/fields'; +import { IndicesOptions } from '../../../../../../common/types/anomaly_detection_jobs'; interface CategoryResults { success: boolean; @@ -20,7 +21,8 @@ export function getCategoryFields( fieldName: string, size: number, query: any, - runtimeMappings?: RuntimeMappings + runtimeMappings?: RuntimeMappings, + indicesOptions?: IndicesOptions ): Promise { return new Promise((resolve, reject) => { ml.esSearch({ @@ -38,6 +40,7 @@ export function getCategoryFields( }, ...(runtimeMappings !== undefined ? { runtime_mappings: runtimeMappings } : {}), }, + ...(indicesOptions ?? {}), }) .then((resp: any) => { const catFields = get(resp, ['aggregations', 'catFields', 'buckets'], []); diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/advanced_job_creator.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/advanced_job_creator.ts index d03d67e058bfa..da5cfc53b7950 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/advanced_job_creator.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/advanced_job_creator.ts @@ -177,16 +177,13 @@ export class AdvancedJobCreator extends JobCreator { // load the start and end times for the selected index // and apply them to the job creator public async autoSetTimeRange() { - try { - const { start, end } = await ml.getTimeFieldRange({ - index: this._indexPatternTitle, - timeFieldName: this.timeFieldName, - query: this.query, - }); - this.setTimeRange(start.epoch, end.epoch); - } catch (error) { - throw Error(error); - } + const { start, end } = await ml.getTimeFieldRange({ + index: this._indexPatternTitle, + timeFieldName: this.timeFieldName, + query: this.query, + indicesOptions: this.datafeedConfig.indices_options, + }); + this.setTimeRange(start.epoch, end.epoch); } public cloneFromExistingJob(job: Job, datafeed: Datafeed) { diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts index 06d489ee5a437..641eda3dbf3e8 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts @@ -51,7 +51,8 @@ export class CategorizationExamplesLoader { this._jobCreator.start, this._jobCreator.end, analyzer, - this._jobCreator.runtimeMappings ?? undefined + this._jobCreator.runtimeMappings ?? undefined, + this._jobCreator.datafeedConfig.indices_options ); return resp; } diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/results_loader/results_loader.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/results_loader/results_loader.ts index c4365bd656f9e..a01581f7526c5 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/results_loader/results_loader.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/results_loader/results_loader.ts @@ -256,7 +256,8 @@ export class ResultsLoader { if (this._jobCreator.splitField !== null) { const fieldValues = await this._chartLoader.loadFieldExampleValues( this._jobCreator.splitField, - this._jobCreator.runtimeMappings + this._jobCreator.runtimeMappings, + this._jobCreator.datafeedConfig.indices_options ); if (fieldValues.length > 0) { this._detectorSplitFieldFilters = { diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/common/json_editor_flyout/json_editor_flyout.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/common/json_editor_flyout/json_editor_flyout.tsx index 39b03ac546081..b2e4a447e4c31 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/common/json_editor_flyout/json_editor_flyout.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/common/json_editor_flyout/json_editor_flyout.tsx @@ -25,7 +25,9 @@ import { CombinedJob, Datafeed } from '../../../../../../../../common/types/anom import { ML_EDITOR_MODE, MLJobEditor } from '../../../../../jobs_list/components/ml_job_editor'; import { isValidJson } from '../../../../../../../../common/util/validation_utils'; import { JobCreatorContext } from '../../job_creator_context'; +import { isAdvancedJobCreator } from '../../../../common/job_creator'; import { DatafeedPreview } from '../datafeed_preview_flyout'; +import { useToastNotificationService } from '../../../../../../services/toast_notification_service'; export enum EDITOR_MODE { HIDDEN, @@ -40,6 +42,7 @@ interface Props { } export const JsonEditorFlyout: FC = ({ isDisabled, jobEditorMode, datafeedEditorMode }) => { const { jobCreator, jobCreatorUpdate, jobCreatorUpdated } = useContext(JobCreatorContext); + const { displayErrorToast } = useToastNotificationService(); const [showJsonFlyout, setShowJsonFlyout] = useState(false); const [showChangedIndicesWarning, setShowChangedIndicesWarning] = useState(false); @@ -120,10 +123,23 @@ export const JsonEditorFlyout: FC = ({ isDisabled, jobEditorMode, datafee setSaveable(valid); } - function onSave() { + async function onSave() { const jobConfig = JSON.parse(jobConfigString); const datafeedConfig = JSON.parse(collapseLiteralStrings(datafeedConfigString)); jobCreator.cloneFromExistingJob(jobConfig, datafeedConfig); + if (isAdvancedJobCreator(jobCreator)) { + try { + await jobCreator.autoSetTimeRange(); + } catch (error) { + const title = i18n.translate( + 'xpack.ml.newJob.wizard.jsonFlyout.autoSetJobCreatorTimeRange.error', + { + defaultMessage: `Error retrieving beginning and end times of index`, + } + ); + displayErrorToast(error, title); + } + } jobCreatorUpdate(); setShowJsonFlyout(false); } diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/bucket_span_estimator/estimate_bucket_span.ts b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/bucket_span_estimator/estimate_bucket_span.ts index f0932b09af46b..85083146c1378 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/bucket_span_estimator/estimate_bucket_span.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/bucket_span_estimator/estimate_bucket_span.ts @@ -9,12 +9,13 @@ import { useContext, useState } from 'react'; import { JobCreatorContext } from '../../../job_creator_context'; import { EVENT_RATE_FIELD_ID } from '../../../../../../../../../common/types/fields'; +import { BucketSpanEstimatorData } from '../../../../../../../../../common/types/job_service'; import { isMultiMetricJobCreator, isPopulationJobCreator, isAdvancedJobCreator, } from '../../../../../common/job_creator'; -import { ml, BucketSpanEstimatorData } from '../../../../../../../services/ml_api_service'; +import { ml } from '../../../../../../../services/ml_api_service'; import { useMlContext } from '../../../../../../../contexts/ml'; import { getToastNotificationService } from '../../../../../../../services/toast_notification_service'; @@ -41,6 +42,7 @@ export function useEstimateBucketSpan() { splitField: undefined, timeField: mlContext.currentIndexPattern.timeFieldName, runtimeMappings: jobCreator.runtimeMappings ?? undefined, + indicesOptions: jobCreator.datafeedConfig.indices_options, }; if ( diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection_summary.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection_summary.tsx index f027372d01204..da9f306cf30e6 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection_summary.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection_summary.tsx @@ -54,7 +54,8 @@ export const CategorizationDetectorsSummary: FC = () => { const resp = await chartLoader.loadEventRateChart( jobCreator.start, jobCreator.end, - chartInterval.getInterval().asMilliseconds() + chartInterval.getInterval().asMilliseconds(), + jobCreator.datafeedConfig.indices_options ); setEventRateChartData(resp); } catch (error) { diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/multi_metric_view/metric_selection.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/multi_metric_view/metric_selection.tsx index 5bf4beacc1593..46eb4b88d0518 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/multi_metric_view/metric_selection.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/multi_metric_view/metric_selection.tsx @@ -111,7 +111,11 @@ export const MultiMetricDetectors: FC = ({ setIsValid }) => { useEffect(() => { if (splitField !== null) { chartLoader - .loadFieldExampleValues(splitField, jobCreator.runtimeMappings) + .loadFieldExampleValues( + splitField, + jobCreator.runtimeMappings, + jobCreator.datafeedConfig.indices_options + ) .then(setFieldValues) .catch((error) => { getToastNotificationService().displayErrorToast(error); @@ -140,7 +144,8 @@ export const MultiMetricDetectors: FC = ({ setIsValid }) => { jobCreator.splitField, fieldValues.length > 0 ? fieldValues[0] : null, cs.intervalMs, - jobCreator.runtimeMappings + jobCreator.runtimeMappings, + jobCreator.datafeedConfig.indices_options ); setLineChartsData(resp); } catch (error) { diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/multi_metric_view/metric_selection_summary.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/multi_metric_view/metric_selection_summary.tsx index 11f2f60e17d3d..a4c344d16482b 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/multi_metric_view/metric_selection_summary.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/multi_metric_view/metric_selection_summary.tsx @@ -43,7 +43,8 @@ export const MultiMetricDetectorsSummary: FC = () => { try { const tempFieldValues = await chartLoader.loadFieldExampleValues( jobCreator.splitField, - jobCreator.runtimeMappings + jobCreator.runtimeMappings, + jobCreator.datafeedConfig.indices_options ); setFieldValues(tempFieldValues); } catch (error) { @@ -76,7 +77,8 @@ export const MultiMetricDetectorsSummary: FC = () => { jobCreator.splitField, fieldValues.length > 0 ? fieldValues[0] : null, cs.intervalMs, - jobCreator.runtimeMappings + jobCreator.runtimeMappings, + jobCreator.datafeedConfig.indices_options ); setLineChartsData(resp); } catch (error) { diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/population_view/metric_selection.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/population_view/metric_selection.tsx index aba2acfa41a85..a7eaaff611183 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/population_view/metric_selection.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/population_view/metric_selection.tsx @@ -160,7 +160,8 @@ export const PopulationDetectors: FC = ({ setIsValid }) => { aggFieldPairList, jobCreator.splitField, cs.intervalMs, - jobCreator.runtimeMappings + jobCreator.runtimeMappings, + jobCreator.datafeedConfig.indices_options ); setLineChartsData(resp); @@ -180,7 +181,11 @@ export const PopulationDetectors: FC = ({ setIsValid }) => { (async (index: number, field: Field) => { return { index, - fields: await chartLoader.loadFieldExampleValues(field, jobCreator.runtimeMappings), + fields: await chartLoader.loadFieldExampleValues( + field, + jobCreator.runtimeMappings, + jobCreator.datafeedConfig.indices_options + ), }; })(i, af.by.field) ); diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/population_view/metric_selection_summary.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/population_view/metric_selection_summary.tsx index c615010891101..55a9d37d1115c 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/population_view/metric_selection_summary.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/population_view/metric_selection_summary.tsx @@ -78,7 +78,8 @@ export const PopulationDetectorsSummary: FC = () => { aggFieldPairList, jobCreator.splitField, cs.intervalMs, - jobCreator.runtimeMappings + jobCreator.runtimeMappings, + jobCreator.datafeedConfig.indices_options ); setLineChartsData(resp); @@ -98,7 +99,11 @@ export const PopulationDetectorsSummary: FC = () => { (async (index: number, field: Field) => { return { index, - fields: await chartLoader.loadFieldExampleValues(field, jobCreator.runtimeMappings), + fields: await chartLoader.loadFieldExampleValues( + field, + jobCreator.runtimeMappings, + jobCreator.datafeedConfig.indices_options + ), }; })(i, af.by.field) ); diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/single_metric_view/metric_selection.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/single_metric_view/metric_selection.tsx index f4a907dcc6a49..0e09a81908e83 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/single_metric_view/metric_selection.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/single_metric_view/metric_selection.tsx @@ -93,7 +93,8 @@ export const SingleMetricDetectors: FC = ({ setIsValid }) => { null, null, cs.intervalMs, - jobCreator.runtimeMappings + jobCreator.runtimeMappings, + jobCreator.datafeedConfig.indices_options ); if (resp[DTR_IDX] !== undefined) { setLineChartData(resp); diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/single_metric_view/metric_selection_summary.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/single_metric_view/metric_selection_summary.tsx index 4d8fc5ef76084..ced94b2095f72 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/single_metric_view/metric_selection_summary.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/single_metric_view/metric_selection_summary.tsx @@ -59,7 +59,8 @@ export const SingleMetricDetectorsSummary: FC = () => { null, null, cs.intervalMs, - jobCreator.runtimeMappings + jobCreator.runtimeMappings, + jobCreator.datafeedConfig.indices_options ); if (resp[DTR_IDX] !== undefined) { setLineChartData(resp); diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/time_range_step/time_range.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/time_range_step/time_range.tsx index bed2d36524e36..d2cf6b7a00471 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/time_range_step/time_range.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/time_range_step/time_range.tsx @@ -47,7 +47,8 @@ export const TimeRangeStep: FC = ({ setCurrentStep, isCurrentStep }) const resp = await chartLoader.loadEventRateChart( jobCreator.start, jobCreator.end, - chartInterval.getInterval().asMilliseconds() + chartInterval.getInterval().asMilliseconds(), + jobCreator.datafeedConfig.indices_options ); setEventRateChartData(resp); } catch (error) { diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx index c8dbb90804444..3a01ce8c70fc8 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx @@ -20,7 +20,6 @@ import { FormattedMessage } from '@kbn/i18n/react'; import { Wizard } from './wizard'; import { WIZARD_STEPS } from '../components/step_types'; import { getJobCreatorTitle } from '../../common/job_creator/util/general'; -import { useMlKibana } from '../../../../contexts/kibana'; import { jobCreatorFactory, isAdvancedJobCreator, @@ -41,6 +40,7 @@ import { ExistingJobsAndGroups, mlJobService } from '../../../../services/job_se import { newJobCapsService } from '../../../../services/new_job_capabilities_service'; import { EVENT_RATE_FIELD_ID } from '../../../../../../common/types/fields'; import { getNewJobDefaults } from '../../../../services/ml_server_info'; +import { useToastNotificationService } from '../../../../services/toast_notification_service'; const PAGE_WIDTH = 1200; // document.querySelector('.single-metric-job-container').width(); const BAR_TARGET = PAGE_WIDTH > 2000 ? 1000 : PAGE_WIDTH / 2; @@ -52,15 +52,13 @@ export interface PageProps { } export const Page: FC = ({ existingJobsAndGroups, jobType }) => { - const { - services: { notifications }, - } = useMlKibana(); const mlContext = useMlContext(); const jobCreator = jobCreatorFactory(jobType)( mlContext.currentIndexPattern, mlContext.currentSavedSearch, mlContext.combinedQuery ); + const { displayErrorToast } = useToastNotificationService(); const { from, to } = getTimeFilterRange(); jobCreator.setTimeRange(from, to); @@ -154,17 +152,12 @@ export const Page: FC = ({ existingJobsAndGroups, jobType }) => { if (autoSetTimeRange && isAdvancedJobCreator(jobCreator)) { // for advanced jobs, load the full time range start and end times // so they can be used for job validation and bucket span estimation - try { - jobCreator.autoSetTimeRange(); - } catch (error) { - const { toasts } = notifications; - toasts.addDanger({ - title: i18n.translate('xpack.ml.newJob.wizard.autoSetJobCreatorTimeRange.error', { - defaultMessage: `Error retrieving beginning and end times of index`, - }), - text: error, + jobCreator.autoSetTimeRange().catch((error) => { + const title = i18n.translate('xpack.ml.newJob.wizard.autoSetJobCreatorTimeRange.error', { + defaultMessage: `Error retrieving beginning and end times of index`, }); - } + displayErrorToast(error, title); + }); } function initCategorizationSettings() { diff --git a/x-pack/plugins/ml/public/application/services/job_service.js b/x-pack/plugins/ml/public/application/services/job_service.js index 33ccf81798353..2fa60b8db83a7 100644 --- a/x-pack/plugins/ml/public/application/services/job_service.js +++ b/x-pack/plugins/ml/public/application/services/job_service.js @@ -13,7 +13,6 @@ import { ml } from './ml_api_service'; import { getToastNotificationService } from '../services/toast_notification_service'; import { isWebUrl } from '../util/url_utils'; -import { ML_DATA_PREVIEW_COUNT } from '../../../common/util/job_utils'; import { TIME_FORMAT } from '../../../common/constants/time_format'; import { parseInterval } from '../../../common/util/parse_interval'; import { validateTimeRange } from '../../../common/util/date_utils'; @@ -348,163 +347,9 @@ class JobService { return job; } - searchPreview(job) { - return new Promise((resolve, reject) => { - if (job.datafeed_config) { - // if query is set, add it to the search, otherwise use match_all - let query = { match_all: {} }; - if (job.datafeed_config.query) { - query = job.datafeed_config.query; - } - - // Get bucket span - // Get first doc time for datafeed - // Create a new query - must user query and must range query. - // Time range 'to' first doc time plus < 10 buckets - - // Do a preliminary search to get the date of the earliest doc matching the - // query in the datafeed. This will be used to apply a time range criteria - // on the datafeed search preview. - // This time filter is required for datafeed searches using aggregations to ensure - // the search does not create too many buckets (default 10000 max_bucket limit), - // but apply it to searches without aggregations too for consistency. - ml.getTimeFieldRange({ - index: job.datafeed_config.indices, - timeFieldName: job.data_description.time_field, - query, - }) - .then((timeRange) => { - const bucketSpan = parseInterval(job.analysis_config.bucket_span); - const earliestMs = timeRange.start.epoch; - const latestMs = +timeRange.start.epoch + 10 * bucketSpan.asMilliseconds(); - - const body = { - query: { - bool: { - must: [ - { - range: { - [job.data_description.time_field]: { - gte: earliestMs, - lt: latestMs, - format: 'epoch_millis', - }, - }, - }, - query, - ], - }, - }, - }; - - // if aggs or aggregations is set, add it to the search - const aggregations = job.datafeed_config.aggs || job.datafeed_config.aggregations; - if (aggregations && Object.keys(aggregations).length) { - body.size = 0; - body.aggregations = aggregations; - - // add script_fields if present - const scriptFields = job.datafeed_config.script_fields; - if (scriptFields && Object.keys(scriptFields).length) { - body.script_fields = scriptFields; - } - - // add runtime_mappings if present - const runtimeMappings = job.datafeed_config.runtime_mappings; - if (runtimeMappings && Object.keys(runtimeMappings).length) { - body.runtime_mappings = runtimeMappings; - } - } else { - // if aggregations is not set and retrieveWholeSource is not set, add all of the fields from the job - body.size = ML_DATA_PREVIEW_COUNT; - - // add script_fields if present - const scriptFields = job.datafeed_config.script_fields; - if (scriptFields && Object.keys(scriptFields).length) { - body.script_fields = scriptFields; - } - - // add runtime_mappings if present - const runtimeMappings = job.datafeed_config.runtime_mappings; - if (runtimeMappings && Object.keys(runtimeMappings).length) { - body.runtime_mappings = runtimeMappings; - } - - const fields = {}; - - // get fields from detectors - if (job.analysis_config.detectors) { - each(job.analysis_config.detectors, (dtr) => { - if (dtr.by_field_name) { - fields[dtr.by_field_name] = {}; - } - if (dtr.field_name) { - fields[dtr.field_name] = {}; - } - if (dtr.over_field_name) { - fields[dtr.over_field_name] = {}; - } - if (dtr.partition_field_name) { - fields[dtr.partition_field_name] = {}; - } - }); - } - - // get fields from influencers - if (job.analysis_config.influencers) { - each(job.analysis_config.influencers, (inf) => { - fields[inf] = {}; - }); - } - - // get fields from categorizationFieldName - if (job.analysis_config.categorization_field_name) { - fields[job.analysis_config.categorization_field_name] = {}; - } - - // get fields from summary_count_field_name - if (job.analysis_config.summary_count_field_name) { - fields[job.analysis_config.summary_count_field_name] = {}; - } - - // get fields from time_field - if (job.data_description.time_field) { - fields[job.data_description.time_field] = {}; - } - - // add runtime fields - if (runtimeMappings) { - Object.keys(runtimeMappings).forEach((fieldName) => { - fields[fieldName] = {}; - }); - } - - const fieldsList = Object.keys(fields); - if (fieldsList.length) { - body.fields = fieldsList; - body._source = false; - } - } - - const data = { - index: job.datafeed_config.indices, - body, - ...(job.datafeed_config.indices_options || {}), - }; - - ml.esSearch(data) - .then((resp) => { - resolve(resp); - }) - .catch((resp) => { - reject(resp); - }); - }) - .catch((resp) => { - reject(resp); - }); - } - }); + searchPreview(combinedJob) { + const { datafeed_config: datafeed, ...job } = combinedJob; + return ml.jobs.datafeedPreview(job, datafeed); } openJob(jobId) { diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts index 8d0ecddaa97b8..e6d0d93cade1f 100644 --- a/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts @@ -24,7 +24,7 @@ import { import { MlCapabilitiesResponse } from '../../../../common/types/capabilities'; import { Calendar, CalendarId, UpdateCalendar } from '../../../../common/types/calendars'; -import { RuntimeMappings } from '../../../../common/types/fields'; +import { BucketSpanEstimatorData } from '../../../../common/types/job_service'; import { Job, JobStats, @@ -33,8 +33,8 @@ import { Detector, AnalysisConfig, ModelSnapshot, + IndicesOptions, } from '../../../../common/types/anomaly_detection_jobs'; -import { ES_AGGREGATION } from '../../../../common/constants/aggregation_types'; import { FieldHistogramRequestConfig, FieldRequestConfig, @@ -53,20 +53,6 @@ export interface MlInfoResponse { cloudId?: string; } -export interface BucketSpanEstimatorData { - aggTypes: Array; - duration: { - start: number; - end: number; - }; - fields: Array; - index: string; - query: any; - splitField: string | undefined; - timeField: string | undefined; - runtimeMappings: RuntimeMappings | undefined; -} - export interface BucketSpanEstimatorResponse { name: string; ms: number; @@ -704,12 +690,14 @@ export function mlApiServicesProvider(httpService: HttpService) { index, timeFieldName, query, + indicesOptions, }: { index: string; timeFieldName?: string; query: any; + indicesOptions?: IndicesOptions; }) { - const body = JSON.stringify({ index, timeFieldName, query }); + const body = JSON.stringify({ index, timeFieldName, query, indicesOptions }); return httpService.http({ path: `${basePath()}/fields_service/time_field_range`, diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/jobs.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/jobs.ts index df72bd25c6bcd..811e9cab365d7 100644 --- a/x-pack/plugins/ml/public/application/services/ml_api_service/jobs.ts +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/jobs.ts @@ -15,6 +15,7 @@ import type { CombinedJobWithStats, Job, Datafeed, + IndicesOptions, } from '../../../../common/types/anomaly_detection_jobs'; import type { JobMessage } from '../../../../common/types/audit_message'; import type { AggFieldNamePair, RuntimeMappings } from '../../../../common/types/fields'; @@ -189,7 +190,8 @@ export const jobsApiProvider = (httpService: HttpService) => ({ aggFieldNamePairs: AggFieldNamePair[], splitFieldName: string | null, splitFieldValue: string | null, - runtimeMappings?: RuntimeMappings + runtimeMappings?: RuntimeMappings, + indicesOptions?: IndicesOptions ) { const body = JSON.stringify({ indexPatternTitle, @@ -202,6 +204,7 @@ export const jobsApiProvider = (httpService: HttpService) => ({ splitFieldName, splitFieldValue, runtimeMappings, + indicesOptions, }); return httpService.http({ path: `${ML_BASE_PATH}/jobs/new_job_line_chart`, @@ -219,7 +222,8 @@ export const jobsApiProvider = (httpService: HttpService) => ({ query: any, aggFieldNamePairs: AggFieldNamePair[], splitFieldName: string, - runtimeMappings?: RuntimeMappings + runtimeMappings?: RuntimeMappings, + indicesOptions?: IndicesOptions ) { const body = JSON.stringify({ indexPatternTitle, @@ -231,6 +235,7 @@ export const jobsApiProvider = (httpService: HttpService) => ({ aggFieldNamePairs, splitFieldName, runtimeMappings, + indicesOptions, }); return httpService.http({ path: `${ML_BASE_PATH}/jobs/new_job_population_chart`, @@ -268,7 +273,8 @@ export const jobsApiProvider = (httpService: HttpService) => ({ start: number, end: number, analyzer: CategorizationAnalyzer, - runtimeMappings?: RuntimeMappings + runtimeMappings?: RuntimeMappings, + indicesOptions?: IndicesOptions ) { const body = JSON.stringify({ indexPatternTitle, @@ -280,6 +286,7 @@ export const jobsApiProvider = (httpService: HttpService) => ({ end, analyzer, runtimeMappings, + indicesOptions, }); return httpService.http<{ examples: CategoryFieldExample[]; @@ -322,4 +329,16 @@ export const jobsApiProvider = (httpService: HttpService) => ({ body, }); }, + + datafeedPreview(job: Job, datafeed: Datafeed) { + const body = JSON.stringify({ job, datafeed }); + return httpService.http<{ + total: number; + categories: Array<{ count?: number; category: Category }>; + }>({ + path: `${ML_BASE_PATH}/jobs/datafeed_preview`, + method: 'POST', + body, + }); + }, }); diff --git a/x-pack/plugins/ml/public/application/services/results_service/results_service.d.ts b/x-pack/plugins/ml/public/application/services/results_service/results_service.d.ts index 90ab302458354..f9a2c1389c828 100644 --- a/x-pack/plugins/ml/public/application/services/results_service/results_service.d.ts +++ b/x-pack/plugins/ml/public/application/services/results_service/results_service.d.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { IndicesOptions } from '../../../../common/types/anomaly_detection_jobs'; import { MlApiServices } from '../ml_api_service'; export function resultsServiceProvider( @@ -58,7 +59,8 @@ export function resultsServiceProvider( timeFieldName: string, earliestMs: number, latestMs: number, - intervalMs: number + intervalMs: number, + indicesOptions?: IndicesOptions ): Promise; getEventDistributionData( index: string, diff --git a/x-pack/plugins/ml/public/application/services/results_service/results_service.js b/x-pack/plugins/ml/public/application/services/results_service/results_service.js index 3390e62030dd6..502692da39c96 100644 --- a/x-pack/plugins/ml/public/application/services/results_service/results_service.js +++ b/x-pack/plugins/ml/public/application/services/results_service/results_service.js @@ -1052,7 +1052,15 @@ export function resultsServiceProvider(mlApiServices) { // Extra query object can be supplied, or pass null if no additional query. // Returned response contains a results property, which is an object // of document counts against time (epoch millis). - getEventRateData(index, query, timeFieldName, earliestMs, latestMs, intervalMs) { + getEventRateData( + index, + query, + timeFieldName, + earliestMs, + latestMs, + intervalMs, + indicesOptions + ) { return new Promise((resolve, reject) => { const obj = { success: true, results: {} }; @@ -1102,6 +1110,7 @@ export function resultsServiceProvider(mlApiServices) { }, }, }, + ...(indicesOptions ?? {}), }) .then((resp) => { const dataByTimeBucket = get(resp, ['aggregations', 'eventRate', 'buckets'], []); diff --git a/x-pack/plugins/ml/server/models/bucket_span_estimator/bucket_span_estimator.d.ts b/x-pack/plugins/ml/server/models/bucket_span_estimator/bucket_span_estimator.d.ts index 40a6bd1decd97..75983975f7acd 100644 --- a/x-pack/plugins/ml/server/models/bucket_span_estimator/bucket_span_estimator.d.ts +++ b/x-pack/plugins/ml/server/models/bucket_span_estimator/bucket_span_estimator.d.ts @@ -8,20 +8,8 @@ import { IScopedClusterClient } from 'kibana/server'; import { ES_AGGREGATION } from '../../../common/constants/aggregation_types'; import { RuntimeMappings } from '../../../common/types/fields'; - -export interface BucketSpanEstimatorData { - aggTypes: Array; - duration: { - start: number; - end: number; - }; - fields: Array; - index: string; - query: any; - splitField: string | undefined; - timeField: string | undefined; - runtimeMappings: RuntimeMappings | undefined; -} +import { IndicesOptions } from '../../../common/types/anomaly_detection_jobs'; +import { BucketSpanEstimatorData } from '../../../common/types/job_service'; export function estimateBucketSpanFactory({ asCurrentUser, diff --git a/x-pack/plugins/ml/server/models/bucket_span_estimator/bucket_span_estimator.js b/x-pack/plugins/ml/server/models/bucket_span_estimator/bucket_span_estimator.js index 79f48645d52f2..29961918e7aba 100644 --- a/x-pack/plugins/ml/server/models/bucket_span_estimator/bucket_span_estimator.js +++ b/x-pack/plugins/ml/server/models/bucket_span_estimator/bucket_span_estimator.js @@ -20,7 +20,17 @@ export function estimateBucketSpanFactory(client) { class BucketSpanEstimator { constructor( - { index, timeField, aggTypes, fields, duration, query, splitField, runtimeMappings }, + { + index, + timeField, + aggTypes, + fields, + duration, + query, + splitField, + runtimeMappings, + indicesOptions, + }, splitFieldValues, maxBuckets ) { @@ -72,7 +82,8 @@ export function estimateBucketSpanFactory(client) { this.index, this.timeField, this.duration, - this.query + this.query, + indicesOptions ); if (this.aggTypes.length === this.fields.length) { @@ -89,7 +100,8 @@ export function estimateBucketSpanFactory(client) { this.duration, this.query, this.thresholds, - this.runtimeMappings + this.runtimeMappings, + indicesOptions ), result: null, }); @@ -112,7 +124,8 @@ export function estimateBucketSpanFactory(client) { this.duration, queryCopy, this.thresholds, - this.runtimeMappings + this.runtimeMappings, + indicesOptions ), result: null, }); @@ -246,7 +259,7 @@ export function estimateBucketSpanFactory(client) { } } - const getFieldCardinality = function (index, field, runtimeMappings) { + const getFieldCardinality = function (index, field, runtimeMappings, indicesOptions) { return new Promise((resolve, reject) => { asCurrentUser .search({ @@ -262,6 +275,7 @@ export function estimateBucketSpanFactory(client) { }, ...(runtimeMappings !== undefined ? { runtime_mappings: runtimeMappings } : {}), }, + ...(indicesOptions ?? {}), }) .then(({ body }) => { const value = get(body, ['aggregations', 'field_count', 'value'], 0); @@ -273,13 +287,13 @@ export function estimateBucketSpanFactory(client) { }); }; - const getRandomFieldValues = function (index, field, query, runtimeMappings) { + const getRandomFieldValues = function (index, field, query, runtimeMappings, indicesOptions) { let fieldValues = []; return new Promise((resolve, reject) => { const NUM_PARTITIONS = 10; // use a partitioned search to load 10 random fields // load ten fields, to test that there are at least 10. - getFieldCardinality(index, field) + getFieldCardinality(index, field, runtimeMappings, indicesOptions) .then((value) => { const numPartitions = Math.floor(value / NUM_PARTITIONS) || 1; asCurrentUser @@ -301,6 +315,7 @@ export function estimateBucketSpanFactory(client) { }, ...(runtimeMappings !== undefined ? { runtime_mappings: runtimeMappings } : {}), }, + ...(indicesOptions ?? {}), }) .then(({ body }) => { // eslint-disable-next-line camelcase @@ -390,7 +405,8 @@ export function estimateBucketSpanFactory(client) { formConfig.index, formConfig.splitField, formConfig.query, - formConfig.runtimeMappings + formConfig.runtimeMappings, + formConfig.indicesOptions ) .then((splitFieldValues) => { runEstimator(splitFieldValues); diff --git a/x-pack/plugins/ml/server/models/bucket_span_estimator/bucket_span_estimator.test.ts b/x-pack/plugins/ml/server/models/bucket_span_estimator/bucket_span_estimator.test.ts index aa576d1f69915..de5514ed1e18f 100644 --- a/x-pack/plugins/ml/server/models/bucket_span_estimator/bucket_span_estimator.test.ts +++ b/x-pack/plugins/ml/server/models/bucket_span_estimator/bucket_span_estimator.test.ts @@ -8,8 +8,9 @@ import { IScopedClusterClient } from 'kibana/server'; import { ES_AGGREGATION } from '../../../common/constants/aggregation_types'; +import { BucketSpanEstimatorData } from '../../../common/types/job_service'; -import { estimateBucketSpanFactory, BucketSpanEstimatorData } from './bucket_span_estimator'; +import { estimateBucketSpanFactory } from './bucket_span_estimator'; const callAs = { search: () => Promise.resolve({ body: {} }), @@ -36,6 +37,7 @@ const formConfig: BucketSpanEstimatorData = { splitField: undefined, timeField: undefined, runtimeMappings: undefined, + indicesOptions: undefined, }; describe('ML - BucketSpanEstimator', () => { diff --git a/x-pack/plugins/ml/server/models/bucket_span_estimator/polled_data_checker.js b/x-pack/plugins/ml/server/models/bucket_span_estimator/polled_data_checker.js index 59da334a18393..8a40787f44490 100644 --- a/x-pack/plugins/ml/server/models/bucket_span_estimator/polled_data_checker.js +++ b/x-pack/plugins/ml/server/models/bucket_span_estimator/polled_data_checker.js @@ -15,11 +15,12 @@ import { get } from 'lodash'; export function polledDataCheckerFactory({ asCurrentUser }) { class PolledDataChecker { - constructor(index, timeField, duration, query) { + constructor(index, timeField, duration, query, indicesOptions) { this.index = index; this.timeField = timeField; this.duration = duration; this.query = query; + this.indicesOptions = indicesOptions; this.isPolled = false; this.minimumBucketSpan = 0; @@ -73,6 +74,7 @@ export function polledDataCheckerFactory({ asCurrentUser }) { index: this.index, size: 0, body: searchBody, + ...(this.indicesOptions ?? {}), }); return body; } diff --git a/x-pack/plugins/ml/server/models/bucket_span_estimator/single_series_checker.js b/x-pack/plugins/ml/server/models/bucket_span_estimator/single_series_checker.js index 25c87c5c2acbf..f9f01070f2f82 100644 --- a/x-pack/plugins/ml/server/models/bucket_span_estimator/single_series_checker.js +++ b/x-pack/plugins/ml/server/models/bucket_span_estimator/single_series_checker.js @@ -18,7 +18,17 @@ export function singleSeriesCheckerFactory({ asCurrentUser }) { const REF_DATA_INTERVAL = { name: '1h', ms: 3600000 }; class SingleSeriesChecker { - constructor(index, timeField, aggType, field, duration, query, thresholds, runtimeMappings) { + constructor( + index, + timeField, + aggType, + field, + duration, + query, + thresholds, + runtimeMappings, + indicesOptions + ) { this.index = index; this.timeField = timeField; this.aggType = aggType; @@ -32,6 +42,7 @@ export function singleSeriesCheckerFactory({ asCurrentUser }) { created: false, }; this.runtimeMappings = runtimeMappings; + this.indicesOptions = indicesOptions; this.interval = null; } @@ -193,6 +204,7 @@ export function singleSeriesCheckerFactory({ asCurrentUser }) { index: this.index, size: 0, body: searchBody, + ...(this.indicesOptions ?? {}), }); return body; } diff --git a/x-pack/plugins/ml/server/models/fields_service/fields_service.ts b/x-pack/plugins/ml/server/models/fields_service/fields_service.ts index 56eddf9df2e04..1270cc6f08e23 100644 --- a/x-pack/plugins/ml/server/models/fields_service/fields_service.ts +++ b/x-pack/plugins/ml/server/models/fields_service/fields_service.ts @@ -13,7 +13,7 @@ import { initCardinalityFieldsCache } from './fields_aggs_cache'; import { AggCardinality } from '../../../common/types/fields'; import { isValidAggregationField } from '../../../common/util/validation_utils'; import { getDatafeedAggregations } from '../../../common/util/datafeed_utils'; -import { Datafeed } from '../../../common/types/anomaly_detection_jobs'; +import { Datafeed, IndicesOptions } from '../../../common/types/anomaly_detection_jobs'; /** * Service for carrying out queries to obtain data @@ -183,6 +183,7 @@ export function fieldsServiceProvider({ asCurrentUser }: IScopedClusterClient) { } = await asCurrentUser.search({ index, body, + ...(datafeedConfig?.indices_options ?? {}), }); if (!aggregations) { @@ -210,7 +211,8 @@ export function fieldsServiceProvider({ asCurrentUser }: IScopedClusterClient) { async function getTimeFieldRange( index: string[] | string, timeFieldName: string, - query: any + query: any, + indicesOptions?: IndicesOptions ): Promise<{ success: boolean; start: { epoch: number; string: string }; @@ -238,6 +240,7 @@ export function fieldsServiceProvider({ asCurrentUser }: IScopedClusterClient) { }, }, }, + ...(indicesOptions ?? {}), }); if (aggregations && aggregations.earliest && aggregations.latest) { @@ -394,6 +397,7 @@ export function fieldsServiceProvider({ asCurrentUser }: IScopedClusterClient) { } = await asCurrentUser.search({ index, body, + ...(datafeedConfig?.indices_options ?? {}), }); if (!aggregations) { diff --git a/x-pack/plugins/ml/server/models/job_service/datafeeds.ts b/x-pack/plugins/ml/server/models/job_service/datafeeds.ts index 88c4659198727..cb651a0a410af 100644 --- a/x-pack/plugins/ml/server/models/job_service/datafeeds.ts +++ b/x-pack/plugins/ml/server/models/job_service/datafeeds.ts @@ -6,10 +6,15 @@ */ import { i18n } from '@kbn/i18n'; +import { IScopedClusterClient } from 'kibana/server'; import { JOB_STATE, DATAFEED_STATE } from '../../../common/constants/states'; import { fillResultsWithTimeouts, isRequestTimeout } from './error_utils'; -import { Datafeed, DatafeedStats } from '../../../common/types/anomaly_detection_jobs'; +import { Datafeed, DatafeedStats, Job } from '../../../common/types/anomaly_detection_jobs'; +import { ML_DATA_PREVIEW_COUNT } from '../../../common/util/job_utils'; +import { fieldsServiceProvider } from '../fields_service'; import type { MlClient } from '../../lib/ml_client'; +import { parseInterval } from '../../../common/util/parse_interval'; +import { isPopulatedObject } from '../../../common/util/object_utils'; export interface MlDatafeedsResponse { datafeeds: Datafeed[]; @@ -27,7 +32,7 @@ interface Results { }; } -export function datafeedsProvider(mlClient: MlClient) { +export function datafeedsProvider(client: IScopedClusterClient, mlClient: MlClient) { async function forceStartDatafeeds(datafeedIds: string[], start?: number, end?: number) { const jobIds = await getJobIdsByDatafeedId(); const doStartsCalled = datafeedIds.reduce((acc, cur) => { @@ -204,6 +209,153 @@ export function datafeedsProvider(mlClient: MlClient) { } } + async function datafeedPreview(job: Job, datafeed: Datafeed) { + let query: any = { match_all: {} }; + if (datafeed.query) { + query = datafeed.query; + } + const { getTimeFieldRange } = fieldsServiceProvider(client); + const { start } = await getTimeFieldRange( + datafeed.indices, + job.data_description.time_field, + query, + datafeed.indices_options + ); + + // Get bucket span + // Get first doc time for datafeed + // Create a new query - must user query and must range query. + // Time range 'to' first doc time plus < 10 buckets + + // Do a preliminary search to get the date of the earliest doc matching the + // query in the datafeed. This will be used to apply a time range criteria + // on the datafeed search preview. + // This time filter is required for datafeed searches using aggregations to ensure + // the search does not create too many buckets (default 10000 max_bucket limit), + // but apply it to searches without aggregations too for consistency. + const bucketSpan = parseInterval(job.analysis_config.bucket_span); + if (bucketSpan === null) { + return; + } + const earliestMs = start.epoch; + const latestMs = +start.epoch + 10 * bucketSpan.asMilliseconds(); + + const body: any = { + query: { + bool: { + must: [ + { + range: { + [job.data_description.time_field]: { + gte: earliestMs, + lt: latestMs, + format: 'epoch_millis', + }, + }, + }, + query, + ], + }, + }, + }; + + // if aggs or aggregations is set, add it to the search + const aggregations = datafeed.aggs ?? datafeed.aggregations; + if (isPopulatedObject(aggregations)) { + body.size = 0; + body.aggregations = aggregations; + + // add script_fields if present + const scriptFields = datafeed.script_fields; + if (isPopulatedObject(scriptFields)) { + body.script_fields = scriptFields; + } + + // add runtime_mappings if present + const runtimeMappings = datafeed.runtime_mappings; + if (isPopulatedObject(runtimeMappings)) { + body.runtime_mappings = runtimeMappings; + } + } else { + // if aggregations is not set and retrieveWholeSource is not set, add all of the fields from the job + body.size = ML_DATA_PREVIEW_COUNT; + + // add script_fields if present + const scriptFields = datafeed.script_fields; + if (isPopulatedObject(scriptFields)) { + body.script_fields = scriptFields; + } + + // add runtime_mappings if present + const runtimeMappings = datafeed.runtime_mappings; + if (isPopulatedObject(runtimeMappings)) { + body.runtime_mappings = runtimeMappings; + } + + const fields = new Set(); + + // get fields from detectors + if (job.analysis_config.detectors) { + job.analysis_config.detectors.forEach((dtr) => { + if (dtr.by_field_name) { + fields.add(dtr.by_field_name); + } + if (dtr.field_name) { + fields.add(dtr.field_name); + } + if (dtr.over_field_name) { + fields.add(dtr.over_field_name); + } + if (dtr.partition_field_name) { + fields.add(dtr.partition_field_name); + } + }); + } + + // get fields from influencers + if (job.analysis_config.influencers) { + job.analysis_config.influencers.forEach((inf) => { + fields.add(inf); + }); + } + + // get fields from categorizationFieldName + if (job.analysis_config.categorization_field_name) { + fields.add(job.analysis_config.categorization_field_name); + } + + // get fields from summary_count_field_name + if (job.analysis_config.summary_count_field_name) { + fields.add(job.analysis_config.summary_count_field_name); + } + + // get fields from time_field + if (job.data_description.time_field) { + fields.add(job.data_description.time_field); + } + + // add runtime fields + if (runtimeMappings) { + Object.keys(runtimeMappings).forEach((fieldName) => { + fields.add(fieldName); + }); + } + + const fieldsList = [...fields]; + if (fieldsList.length) { + body.fields = fieldsList; + body._source = false; + } + } + const data = { + index: datafeed.indices, + body, + ...(datafeed.indices_options ?? {}), + }; + + return (await client.asCurrentUser.search(data)).body; + } + return { forceStartDatafeeds, stopDatafeeds, @@ -211,5 +363,6 @@ export function datafeedsProvider(mlClient: MlClient) { getDatafeedIdsByJobId, getJobIdsByDatafeedId, getDatafeedByJobId, + datafeedPreview, }; } diff --git a/x-pack/plugins/ml/server/models/job_service/index.ts b/x-pack/plugins/ml/server/models/job_service/index.ts index e88ff8cb751aa..d36ec822c1314 100644 --- a/x-pack/plugins/ml/server/models/job_service/index.ts +++ b/x-pack/plugins/ml/server/models/job_service/index.ts @@ -16,12 +16,12 @@ import type { MlClient } from '../../lib/ml_client'; export function jobServiceProvider(client: IScopedClusterClient, mlClient: MlClient) { return { - ...datafeedsProvider(mlClient), + ...datafeedsProvider(client, mlClient), ...jobsProvider(client, mlClient), ...groupsProvider(mlClient), ...newJobCapsProvider(client), ...newJobChartsProvider(client), ...topCategoriesProvider(mlClient), - ...modelSnapshotProvider(mlClient), + ...modelSnapshotProvider(client, mlClient), }; } diff --git a/x-pack/plugins/ml/server/models/job_service/jobs.ts b/x-pack/plugins/ml/server/models/job_service/jobs.ts index dc2c04540ef21..ac3e00a918da8 100644 --- a/x-pack/plugins/ml/server/models/job_service/jobs.ts +++ b/x-pack/plugins/ml/server/models/job_service/jobs.ts @@ -52,6 +52,7 @@ export function jobsProvider(client: IScopedClusterClient, mlClient: MlClient) { const { asInternalUser } = client; const { forceDeleteDatafeed, getDatafeedIdsByJobId, getDatafeedByJobId } = datafeedsProvider( + client, mlClient ); const { getAuditMessagesSummary } = jobAuditMessagesProvider(client, mlClient); diff --git a/x-pack/plugins/ml/server/models/job_service/model_snapshots.ts b/x-pack/plugins/ml/server/models/job_service/model_snapshots.ts index 9f4607414c10a..f1f5d98b96a53 100644 --- a/x-pack/plugins/ml/server/models/job_service/model_snapshots.ts +++ b/x-pack/plugins/ml/server/models/job_service/model_snapshots.ts @@ -7,6 +7,7 @@ import Boom from '@hapi/boom'; import { i18n } from '@kbn/i18n'; +import { IScopedClusterClient } from 'kibana/server'; import { ModelSnapshot } from '../../../common/types/anomaly_detection_jobs'; import { datafeedsProvider } from './datafeeds'; import { FormCalendar, CalendarManager } from '../calendar'; @@ -20,8 +21,8 @@ export interface RevertModelSnapshotResponse { model: ModelSnapshot; } -export function modelSnapshotProvider(mlClient: MlClient) { - const { forceStartDatafeeds, getDatafeedIdsByJobId } = datafeedsProvider(mlClient); +export function modelSnapshotProvider(client: IScopedClusterClient, mlClient: MlClient) { + const { forceStartDatafeeds, getDatafeedIdsByJobId } = datafeedsProvider(client, mlClient); async function revertModelSnapshot( jobId: string, diff --git a/x-pack/plugins/ml/server/models/job_service/new_job/categorization/examples.ts b/x-pack/plugins/ml/server/models/job_service/new_job/categorization/examples.ts index 63df425791e85..37fa675362773 100644 --- a/x-pack/plugins/ml/server/models/job_service/new_job/categorization/examples.ts +++ b/x-pack/plugins/ml/server/models/job_service/new_job/categorization/examples.ts @@ -15,6 +15,7 @@ import { CategoryFieldExample, } from '../../../../../common/types/categories'; import { RuntimeMappings } from '../../../../../common/types/fields'; +import { IndicesOptions } from '../../../../../common/types/anomaly_detection_jobs'; import { ValidationResults } from './validation_results'; const CHUNK_SIZE = 100; @@ -34,7 +35,8 @@ export function categorizationExamplesProvider({ start: number, end: number, analyzer: CategorizationAnalyzer, - runtimeMappings: RuntimeMappings | undefined + runtimeMappings: RuntimeMappings | undefined, + indicesOptions: IndicesOptions | undefined ): Promise<{ examples: CategoryFieldExample[]; error?: any }> { if (timeField !== undefined) { const range = { @@ -69,6 +71,7 @@ export function categorizationExamplesProvider({ sort: ['_doc'], ...(runtimeMappings !== undefined ? { runtime_mappings: runtimeMappings } : {}), }, + ...(indicesOptions ?? {}), }); // hit.fields can be undefined if value is originally null @@ -169,7 +172,8 @@ export function categorizationExamplesProvider({ start: number, end: number, analyzer: CategorizationAnalyzer, - runtimeMappings: RuntimeMappings | undefined + runtimeMappings: RuntimeMappings | undefined, + indicesOptions: IndicesOptions | undefined ) { const resp = await categorizationExamples( indexPatternTitle, @@ -180,7 +184,8 @@ export function categorizationExamplesProvider({ start, end, analyzer, - runtimeMappings + runtimeMappings, + indicesOptions ); const { examples } = resp; diff --git a/x-pack/plugins/ml/server/models/job_service/new_job/line_chart.ts b/x-pack/plugins/ml/server/models/job_service/new_job/line_chart.ts index c83485211b455..e6a2432e28dc1 100644 --- a/x-pack/plugins/ml/server/models/job_service/new_job/line_chart.ts +++ b/x-pack/plugins/ml/server/models/job_service/new_job/line_chart.ts @@ -12,6 +12,7 @@ import { EVENT_RATE_FIELD_ID, RuntimeMappings, } from '../../../../common/types/fields'; +import { IndicesOptions } from '../../../../common/types/anomaly_detection_jobs'; import { ML_MEDIAN_PERCENTS } from '../../../../common/util/job_utils'; type DtrIndex = number; @@ -39,7 +40,8 @@ export function newJobLineChartProvider({ asCurrentUser }: IScopedClusterClient) aggFieldNamePairs: AggFieldNamePair[], splitFieldName: string | null, splitFieldValue: string | null, - runtimeMappings: RuntimeMappings | undefined + runtimeMappings: RuntimeMappings | undefined, + indicesOptions: IndicesOptions | undefined ) { const json: object = getSearchJsonFromConfig( indexPatternTitle, @@ -51,7 +53,8 @@ export function newJobLineChartProvider({ asCurrentUser }: IScopedClusterClient) aggFieldNamePairs, splitFieldName, splitFieldValue, - runtimeMappings + runtimeMappings, + indicesOptions ); const { body } = await asCurrentUser.search(json); @@ -110,7 +113,8 @@ function getSearchJsonFromConfig( aggFieldNamePairs: AggFieldNamePair[], splitFieldName: string | null, splitFieldValue: string | null, - runtimeMappings: RuntimeMappings | undefined + runtimeMappings: RuntimeMappings | undefined, + indicesOptions: IndicesOptions | undefined ): object { const json = { index: indexPatternTitle, @@ -134,6 +138,7 @@ function getSearchJsonFromConfig( }, ...(runtimeMappings !== undefined ? { runtime_mappings: runtimeMappings } : {}), }, + ...(indicesOptions ?? {}), }; if (query.bool === undefined) { diff --git a/x-pack/plugins/ml/server/models/job_service/new_job/population_chart.ts b/x-pack/plugins/ml/server/models/job_service/new_job/population_chart.ts index 10f6d94e764ac..2385ffef67191 100644 --- a/x-pack/plugins/ml/server/models/job_service/new_job/population_chart.ts +++ b/x-pack/plugins/ml/server/models/job_service/new_job/population_chart.ts @@ -12,6 +12,7 @@ import { EVENT_RATE_FIELD_ID, RuntimeMappings, } from '../../../../common/types/fields'; +import { IndicesOptions } from '../../../../common/types/anomaly_detection_jobs'; import { ML_MEDIAN_PERCENTS } from '../../../../common/util/job_utils'; const OVER_FIELD_EXAMPLES_COUNT = 40; @@ -44,7 +45,8 @@ export function newJobPopulationChartProvider({ asCurrentUser }: IScopedClusterC query: object, aggFieldNamePairs: AggFieldNamePair[], splitFieldName: string | null, - runtimeMappings: RuntimeMappings | undefined + runtimeMappings: RuntimeMappings | undefined, + indicesOptions: IndicesOptions | undefined ) { const json: object = getPopulationSearchJsonFromConfig( indexPatternTitle, @@ -55,7 +57,8 @@ export function newJobPopulationChartProvider({ asCurrentUser }: IScopedClusterC query, aggFieldNamePairs, splitFieldName, - runtimeMappings + runtimeMappings, + indicesOptions ); const { body } = await asCurrentUser.search(json); @@ -138,7 +141,8 @@ function getPopulationSearchJsonFromConfig( query: any, aggFieldNamePairs: AggFieldNamePair[], splitFieldName: string | null, - runtimeMappings: RuntimeMappings | undefined + runtimeMappings: RuntimeMappings | undefined, + indicesOptions: IndicesOptions | undefined ): object { const json = { index: indexPatternTitle, @@ -162,6 +166,7 @@ function getPopulationSearchJsonFromConfig( }, ...(runtimeMappings !== undefined ? { runtime_mappings: runtimeMappings } : {}), }, + ...(indicesOptions ?? {}), }; if (query.bool === undefined) { diff --git a/x-pack/plugins/ml/server/routes/apidoc.json b/x-pack/plugins/ml/server/routes/apidoc.json index 33ae5b4f96829..1a10046380658 100644 --- a/x-pack/plugins/ml/server/routes/apidoc.json +++ b/x-pack/plugins/ml/server/routes/apidoc.json @@ -84,6 +84,7 @@ "GetLookBackProgress", "ValidateCategoryExamples", "TopCategories", + "DatafeedPreview", "UpdateGroups", "DeletingJobTasks", "DeleteJobs", diff --git a/x-pack/plugins/ml/server/routes/apidoc_scripts/schema_extractor.test.ts b/x-pack/plugins/ml/server/routes/apidoc_scripts/schema_extractor.test.ts index b7a706a9bbd0c..6c15e6c707a5a 100644 --- a/x-pack/plugins/ml/server/routes/apidoc_scripts/schema_extractor.test.ts +++ b/x-pack/plugins/ml/server/routes/apidoc_scripts/schema_extractor.test.ts @@ -133,7 +133,7 @@ describe('schema_extractor', () => { { name: 'expand_wildcards', documentation: '', - type: 'string[]', + type: '"all" | "open" | "closed" | "hidden" | "none"[]', }, { name: 'ignore_unavailable', diff --git a/x-pack/plugins/ml/server/routes/fields_service.ts b/x-pack/plugins/ml/server/routes/fields_service.ts index 6e68a80354491..c087b86172fa9 100644 --- a/x-pack/plugins/ml/server/routes/fields_service.ts +++ b/x-pack/plugins/ml/server/routes/fields_service.ts @@ -22,8 +22,8 @@ function getCardinalityOfFields(client: IScopedClusterClient, payload: any) { function getTimeFieldRange(client: IScopedClusterClient, payload: any) { const fs = fieldsServiceProvider(client); - const { index, timeFieldName, query } = payload; - return fs.getTimeFieldRange(index, timeFieldName, query); + const { index, timeFieldName, query, indicesOptions } = payload; + return fs.getTimeFieldRange(index, timeFieldName, query, indicesOptions); } /** diff --git a/x-pack/plugins/ml/server/routes/job_service.ts b/x-pack/plugins/ml/server/routes/job_service.ts index 1e028dfb20b4d..b3aa9f956895a 100644 --- a/x-pack/plugins/ml/server/routes/job_service.ts +++ b/x-pack/plugins/ml/server/routes/job_service.ts @@ -20,12 +20,14 @@ import { updateGroupsSchema, revertModelSnapshotSchema, jobsExistSchema, + datafeedPreviewSchema, } from './schemas/job_service_schema'; import { jobIdSchema } from './schemas/anomaly_detectors_schema'; import { jobServiceProvider } from '../models/job_service'; import { categorizationExamplesProvider } from '../models/job_service/new_job'; +import { getAuthorizationHeader } from '../lib/request_authorization'; /** * Routes for job service @@ -535,6 +537,7 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { splitFieldName, splitFieldValue, runtimeMappings, + indicesOptions, } = request.body; const { newJobLineChart } = jobServiceProvider(client, mlClient); @@ -548,7 +551,8 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { aggFieldNamePairs, splitFieldName, splitFieldValue, - runtimeMappings + runtimeMappings, + indicesOptions ); return response.ok({ @@ -591,6 +595,7 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { aggFieldNamePairs, splitFieldName, runtimeMappings, + indicesOptions, } = request.body; const { newJobPopulationChart } = jobServiceProvider(client, mlClient); @@ -603,7 +608,8 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { query, aggFieldNamePairs, splitFieldName, - runtimeMappings + runtimeMappings, + indicesOptions ); return response.ok({ @@ -710,6 +716,7 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { end, analyzer, runtimeMappings, + indicesOptions, } = request.body; const resp = await validateCategoryExamples( @@ -721,7 +728,8 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { start, end, analyzer, - runtimeMappings + runtimeMappings, + indicesOptions ); return response.ok({ @@ -767,6 +775,52 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { }) ); + /** + * @apiGroup JobService + * + * @api {post} /api/ml/jobs/datafeed_preview Get datafeed preview + * @apiName DatafeedPreview + * @apiDescription Returns a preview of the datafeed search + * + * @apiSchema (body) datafeedPreviewSchema + */ + router.post( + { + path: '/api/ml/jobs/datafeed_preview', + validate: { + body: datafeedPreviewSchema, + }, + options: { + tags: ['access:ml:canGetJobs'], + }, + }, + routeGuard.fullLicenseAPIGuard(async ({ client, mlClient, request, response }) => { + try { + const { datafeedId, job, datafeed } = request.body; + + if (datafeedId !== undefined) { + const { body } = await mlClient.previewDatafeed( + { + datafeed_id: datafeedId, + }, + getAuthorizationHeader(request) + ); + return response.ok({ + body, + }); + } + + const { datafeedPreview } = jobServiceProvider(client, mlClient); + const body = await datafeedPreview(job, datafeed); + return response.ok({ + body, + }); + } catch (e) { + return response.customError(wrapError(e)); + } + }) + ); + /** * @apiGroup JobService * diff --git a/x-pack/plugins/ml/server/routes/schemas/datafeeds_schema.ts b/x-pack/plugins/ml/server/routes/schemas/datafeeds_schema.ts index d8d0cd659c2e6..e860be59e4eaf 100644 --- a/x-pack/plugins/ml/server/routes/schemas/datafeeds_schema.ts +++ b/x-pack/plugins/ml/server/routes/schemas/datafeeds_schema.ts @@ -13,6 +13,23 @@ export const startDatafeedSchema = schema.object({ timeout: schema.maybe(schema.any()), }); +export const indicesOptionsSchema = schema.object({ + expand_wildcards: schema.maybe( + schema.arrayOf( + schema.oneOf([ + schema.literal('all'), + schema.literal('open'), + schema.literal('closed'), + schema.literal('hidden'), + schema.literal('none'), + ]) + ) + ), + ignore_unavailable: schema.maybe(schema.boolean()), + allow_no_indices: schema.maybe(schema.boolean()), + ignore_throttled: schema.maybe(schema.boolean()), +}); + export const datafeedConfigSchema = schema.object({ datafeed_id: schema.maybe(schema.string()), feed_id: schema.maybe(schema.string()), @@ -35,14 +52,7 @@ export const datafeedConfigSchema = schema.object({ runtime_mappings: schema.maybe(schema.any()), scroll_size: schema.maybe(schema.number()), delayed_data_check_config: schema.maybe(schema.any()), - indices_options: schema.maybe( - schema.object({ - expand_wildcards: schema.maybe(schema.arrayOf(schema.string())), - ignore_unavailable: schema.maybe(schema.boolean()), - allow_no_indices: schema.maybe(schema.boolean()), - ignore_throttled: schema.maybe(schema.boolean()), - }) - ), + indices_options: indicesOptionsSchema, }); export const datafeedIdSchema = schema.object({ datafeedId: schema.string() }); diff --git a/x-pack/plugins/ml/server/routes/schemas/fields_service_schema.ts b/x-pack/plugins/ml/server/routes/schemas/fields_service_schema.ts index 462fca17bda85..db827b26fe73a 100644 --- a/x-pack/plugins/ml/server/routes/schemas/fields_service_schema.ts +++ b/x-pack/plugins/ml/server/routes/schemas/fields_service_schema.ts @@ -6,6 +6,7 @@ */ import { schema } from '@kbn/config-schema'; +import { indicesOptionsSchema } from './datafeeds_schema'; export const getCardinalityOfFieldsSchema = schema.object({ /** Index or indexes for which to return the time range. */ @@ -29,4 +30,6 @@ export const getTimeFieldRangeSchema = schema.object({ timeFieldName: schema.maybe(schema.string()), /** Query to match documents in the index(es). */ query: schema.maybe(schema.any()), + /** Additional search options. */ + indicesOptions: indicesOptionsSchema, }); diff --git a/x-pack/plugins/ml/server/routes/schemas/job_service_schema.ts b/x-pack/plugins/ml/server/routes/schemas/job_service_schema.ts index 65955fbc47a37..8e160094c68eb 100644 --- a/x-pack/plugins/ml/server/routes/schemas/job_service_schema.ts +++ b/x-pack/plugins/ml/server/routes/schemas/job_service_schema.ts @@ -6,6 +6,8 @@ */ import { schema } from '@kbn/config-schema'; +import { anomalyDetectionJobSchema } from './anomaly_detectors_schema'; +import { datafeedConfigSchema, indicesOptionsSchema } from './datafeeds_schema'; export const categorizationFieldExamplesSchema = { indexPatternTitle: schema.string(), @@ -17,6 +19,7 @@ export const categorizationFieldExamplesSchema = { end: schema.number(), analyzer: schema.any(), runtimeMappings: schema.maybe(schema.any()), + indicesOptions: indicesOptionsSchema, }; export const chartSchema = { @@ -30,6 +33,7 @@ export const chartSchema = { splitFieldName: schema.maybe(schema.nullable(schema.string())), splitFieldValue: schema.maybe(schema.nullable(schema.string())), runtimeMappings: schema.maybe(schema.any()), + indicesOptions: indicesOptionsSchema, }; export const datafeedIdsSchema = schema.object({ @@ -92,6 +96,16 @@ export const revertModelSnapshotSchema = schema.object({ ), }); +export const datafeedPreviewSchema = schema.oneOf([ + schema.object({ + job: schema.maybe(schema.object(anomalyDetectionJobSchema)), + datafeed: schema.maybe(datafeedConfigSchema), + }), + schema.object({ + datafeedId: schema.maybe(schema.string()), + }), +]); + export const jobsExistSchema = schema.object({ jobIds: schema.arrayOf(schema.string()), allSpaces: schema.maybe(schema.boolean()), diff --git a/x-pack/plugins/ml/server/routes/schemas/job_validation_schema.ts b/x-pack/plugins/ml/server/routes/schemas/job_validation_schema.ts index 8c054d54e0589..ad2bafdfb5dd1 100644 --- a/x-pack/plugins/ml/server/routes/schemas/job_validation_schema.ts +++ b/x-pack/plugins/ml/server/routes/schemas/job_validation_schema.ts @@ -7,7 +7,7 @@ import { schema } from '@kbn/config-schema'; import { analysisConfigSchema, anomalyDetectionJobSchema } from './anomaly_detectors_schema'; -import { datafeedConfigSchema } from './datafeeds_schema'; +import { datafeedConfigSchema, indicesOptionsSchema } from './datafeeds_schema'; export const estimateBucketSpanSchema = schema.object({ aggTypes: schema.arrayOf(schema.nullable(schema.string())), @@ -19,6 +19,7 @@ export const estimateBucketSpanSchema = schema.object({ splitField: schema.maybe(schema.string()), timeField: schema.maybe(schema.string()), runtimeMappings: schema.maybe(schema.any()), + indicesOptions: indicesOptionsSchema, }); export const modelMemoryLimitSchema = schema.object({