From 840893271eb260028ae8cb3331e2901fffa9fb18 Mon Sep 17 00:00:00 2001 From: Robert Oskamp Date: Wed, 11 Sep 2019 16:57:23 +0200 Subject: [PATCH] [ML] Add multi metric job wizard test (#45279) This PR adds functional UI tests to create a machine learning job using the multi metric wizard. --- .../charts/anomaly_chart/anomaly_chart.tsx | 2 +- .../detector_title/detector_title.tsx | 4 +- .../multi_metric_view/chart_grid.tsx | 2 +- .../components/multi_metric_view/settings.tsx | 4 +- .../components/split_cards/split_cards.tsx | 29 ++- .../create_multi_metric_job.ts | 214 ++++++++++++++++++ .../create_single_metric_job.ts | 34 +-- .../functional/apps/machine_learning/index.ts | 1 + .../services/machine_learning/job_table.ts | 34 +++ .../machine_learning/job_type_selection.ts | 5 + .../machine_learning/job_wizard_common.ts | 63 +++++- 11 files changed, 354 insertions(+), 38 deletions(-) create mode 100644 x-pack/test/functional/apps/machine_learning/create_multi_metric_job.ts diff --git a/x-pack/legacy/plugins/ml/public/jobs/new_job_new/pages/components/charts/anomaly_chart/anomaly_chart.tsx b/x-pack/legacy/plugins/ml/public/jobs/new_job_new/pages/components/charts/anomaly_chart/anomaly_chart.tsx index bcd16c858069..728229fc3091 100644 --- a/x-pack/legacy/plugins/ml/public/jobs/new_job_new/pages/components/charts/anomaly_chart/anomaly_chart.tsx +++ b/x-pack/legacy/plugins/ml/public/jobs/new_job_new/pages/components/charts/anomaly_chart/anomaly_chart.tsx @@ -43,7 +43,7 @@ export const AnomalyChart: FC = ({ const data = chartType === CHART_TYPE.SCATTER ? flattenData(chartData) : chartData; const xDomain = getXRange(data); return ( -
+
0} loading={loading}> diff --git a/x-pack/legacy/plugins/ml/public/jobs/new_job_new/pages/components/pick_fields_step/components/detector_title/detector_title.tsx b/x-pack/legacy/plugins/ml/public/jobs/new_job_new/pages/components/pick_fields_step/components/detector_title/detector_title.tsx index 3eea38ab1198..040d980cefab 100644 --- a/x-pack/legacy/plugins/ml/public/jobs/new_job_new/pages/components/pick_fields_step/components/detector_title/detector_title.tsx +++ b/x-pack/legacy/plugins/ml/public/jobs/new_job_new/pages/components/pick_fields_step/components/detector_title/detector_title.tsx @@ -33,7 +33,9 @@ export const DetectorTitle: FC = ({ return ( - {getTitle(agg, field, splitField)} + + {getTitle(agg, field, splitField)} + {children !== false && ( diff --git a/x-pack/legacy/plugins/ml/public/jobs/new_job_new/pages/components/pick_fields_step/components/multi_metric_view/chart_grid.tsx b/x-pack/legacy/plugins/ml/public/jobs/new_job_new/pages/components/pick_fields_step/components/multi_metric_view/chart_grid.tsx index 582c6a4a5269..cb2ca459da4f 100644 --- a/x-pack/legacy/plugins/ml/public/jobs/new_job_new/pages/components/pick_fields_step/components/multi_metric_view/chart_grid.tsx +++ b/x-pack/legacy/plugins/ml/public/jobs/new_job_new/pages/components/pick_fields_step/components/multi_metric_view/chart_grid.tsx @@ -54,7 +54,7 @@ export const ChartGrid: FC = ({ > {aggFieldPairList.map((af, i) => ( - + = ({ setIsValid }) => { return ( - + - + diff --git a/x-pack/legacy/plugins/ml/public/jobs/new_job_new/pages/components/pick_fields_step/components/split_cards/split_cards.tsx b/x-pack/legacy/plugins/ml/public/jobs/new_job_new/pages/components/pick_fields_step/components/split_cards/split_cards.tsx index b9c2c6242ec3..a3c13fce53a9 100644 --- a/x-pack/legacy/plugins/ml/public/jobs/new_job_new/pages/components/pick_fields_step/components/split_cards/split_cards.tsx +++ b/x-pack/legacy/plugins/ml/public/jobs/new_job_new/pages/components/pick_fields_step/components/split_cards/split_cards.tsx @@ -70,8 +70,13 @@ export const SplitCards: FC = memo( }; return (
storePanels(ref, marginBottom)} style={style}> - -
{fieldName}
+ +
+ {fieldName} +
); @@ -80,13 +85,16 @@ export const SplitCards: FC = memo( return ( - + {(fieldValues.length === 0 || numberOfDetectors === 0) && {children}} {fieldValues.length > 0 && numberOfDetectors > 0 && splitField !== null && ( {jobType === JOB_TYPE.MULTI_METRIC && ( -
+
= memo( )} {getBackPanels()} - -
{fieldValues[0]}
+ +
+ {fieldValues[0]} +
{children} diff --git a/x-pack/test/functional/apps/machine_learning/create_multi_metric_job.ts b/x-pack/test/functional/apps/machine_learning/create_multi_metric_job.ts new file mode 100644 index 000000000000..2eeb0b199858 --- /dev/null +++ b/x-pack/test/functional/apps/machine_learning/create_multi_metric_job.ts @@ -0,0 +1,214 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import expect from '@kbn/expect'; + +import { FtrProviderContext } from '../../ftr_provider_context'; + +// eslint-disable-next-line import/no-default-export +export default function({ getService }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const ml = getService('ml'); + + const jobId = `fq_multi_1_${Date.now()}`; + const jobDescription = + 'Create multi metric job based on the farequote dataset with 15m bucketspan and min/max/mean(responsetime) split by airline'; + const jobGroups = ['automated', 'farequote', 'multi-metric']; + const aggAndFieldIdentifiers = ['Min(responsetime)', 'Max(responsetime)', 'Mean(responsetime)']; + const splitField = 'airline'; + const bucketSpan = '15m'; + const memoryLimit = '20MB'; + + describe('multi metric job creation', function() { + this.tags(['smoke', 'mlqa']); + before(async () => { + await esArchiver.loadIfNeeded('ml/farequote'); + }); + + after(async () => { + await esArchiver.unload('ml/farequote'); + await ml.api.cleanMlIndices(); + await ml.api.cleanDataframeIndices(); + }); + + it('loads the job management page', async () => { + await ml.navigation.navigateToMl(); + await ml.navigation.navigateToJobManagement(); + }); + + it('loads the new job source selection page', async () => { + await ml.jobManagement.navigateToNewJobSourceSelection(); + }); + + it('loads the job type selection page', async () => { + await ml.jobSourceSelection.selectSourceIndexPattern('farequote'); + }); + + it('loads the single metric job wizard page', async () => { + await ml.jobTypeSelection.selectMultiMetricJob(); + }); + + it('displays the time range step', async () => { + await ml.jobWizardCommon.assertTimeRangeSectionExists(); + }); + + it('displays the event rate chart', async () => { + await ml.jobWizardCommon.clickUseFullDataButton(); + await ml.jobWizardCommon.assertEventRateChartExists(); + }); + + it('displays the pick fields step', async () => { + await ml.jobWizardCommon.clickNextButton(); + await ml.jobWizardCommon.assertPickFieldsSectionExists(); + }); + + it('selects detectors and displays detector previews', async () => { + for (const [index, aggAndFieldIdentifier] of aggAndFieldIdentifiers.entries()) { + await ml.jobWizardCommon.assertAggAndFieldInputExists(); + await ml.jobWizardCommon.selectAggAndField(aggAndFieldIdentifier); + await ml.jobWizardCommon.assertDetectorPreviewExists(aggAndFieldIdentifier, index, 'LINE'); + } + }); + + it('inputs the split field and displays split cards', async () => { + await ml.jobWizardCommon.assertMultiMetricSplitFieldInputExists(); + await ml.jobWizardCommon.selectMultiMetricSplitField(splitField); + await ml.jobWizardCommon.assertMultiMetricSplitFieldSelection(splitField); + + await ml.jobWizardCommon.assertDetectorSplitExists(splitField); + await ml.jobWizardCommon.assertDetectorSplitFrontCardTitle('AAL'); + await ml.jobWizardCommon.assertDetectorSplitNumberOfBackCards(9); + + await ml.jobWizardCommon.assertInfluencerSelection([splitField]); + }); + + it('displays the influencer field', async () => { + await ml.jobWizardCommon.assertInfluencerInputExists(); + await ml.jobWizardCommon.assertInfluencerSelection([splitField]); + }); + + it('inputs the bucket span', async () => { + await ml.jobWizardCommon.assertBucketSpanInputExists(); + await ml.jobWizardCommon.setBucketSpan(bucketSpan); + await ml.jobWizardCommon.assertBucketSpanValue(bucketSpan); + }); + + it('displays the job details step', async () => { + await ml.jobWizardCommon.clickNextButton(); + await ml.jobWizardCommon.assertJobDetailsSectionExists(); + }); + + it('inputs the job id', async () => { + await ml.jobWizardCommon.assertJobIdInputExists(); + await ml.jobWizardCommon.setJobId(jobId); + await ml.jobWizardCommon.assertJobIdValue(jobId); + }); + + it('inputs the job description', async () => { + await ml.jobWizardCommon.assertJobDescriptionInputExists(); + await ml.jobWizardCommon.setJobDescription(jobDescription); + await ml.jobWizardCommon.assertJobDescriptionValue(jobDescription); + }); + + it('inputs job groups', async () => { + await ml.jobWizardCommon.assertJobGroupInputExists(); + for (const jobGroup of jobGroups) { + await ml.jobWizardCommon.addJobGroup(jobGroup); + } + await ml.jobWizardCommon.assertJobGroupSelection(jobGroups); + }); + + it('opens the advanced section', async () => { + await ml.jobWizardCommon.ensureAdvancedSectionOpen(); + }); + + it('displays the model plot switch', async () => { + await ml.jobWizardCommon.assertModelPlotSwitchExists(); + }); + + it('enables the dedicated index switch', async () => { + await ml.jobWizardCommon.assertDedicatedIndexSwitchExists(); + await ml.jobWizardCommon.activateDedicatedIndexSwitch(); + await ml.jobWizardCommon.assertDedicatedIndexSwitchCheckedState(true); + }); + + it('inputs the model memory limit', async () => { + await ml.jobWizardCommon.assertModelMemoryLimitInputExists(); + await ml.jobWizardCommon.setModelMemoryLimit(memoryLimit); + await ml.jobWizardCommon.assertModelMemoryLimitValue(memoryLimit); + }); + + it('displays the validation step', async () => { + await ml.jobWizardCommon.clickNextButton(); + await ml.jobWizardCommon.assertValidationSectionExists(); + }); + + it('displays the summary step', async () => { + await ml.jobWizardCommon.clickNextButton(); + await ml.jobWizardCommon.assertSummarySectionExists(); + }); + + it('creates the job and finishes processing', async () => { + await ml.jobWizardCommon.assertCreateJobButtonExists(); + await ml.jobWizardCommon.createJobAndWaitForCompletion(); + }); + + it('displays the created job in the job list', async () => { + await ml.navigation.navigateToMl(); + await ml.navigation.navigateToJobManagement(); + + await ml.jobTable.waitForJobsToLoad(); + await ml.jobTable.filterWithSearchString(jobId); + const rows = await ml.jobTable.parseJobTable(); + expect(rows.filter(row => row.id === jobId)).to.have.length(1); + }); + + it('displays details for the created job in the job list', async () => { + const expectedRow = { + id: jobId, + description: jobDescription, + jobGroups, + recordCount: '86,274', + memoryStatus: 'ok', + jobState: 'closed', + datafeedState: 'stopped', + latestTimestamp: '2016-02-11 23:59:54', + }; + await ml.jobTable.assertJobRowFields(jobId, expectedRow); + + const expectedCounts = { + job_id: jobId, + processed_record_count: '86,274', + processed_field_count: '172,548', + input_bytes: '6.4 MB', + input_field_count: '172,548', + invalid_date_count: '0', + missing_field_count: '0', + out_of_order_timestamp_count: '0', + empty_bucket_count: '0', + sparse_bucket_count: '0', + bucket_count: '479', + earliest_record_timestamp: '2016-02-07 00:00:00', + latest_record_timestamp: '2016-02-11 23:59:54', + input_record_count: '86,274', + latest_bucket_timestamp: '2016-02-11 23:45:00', + }; + const expectedModelSizeStats = { + job_id: jobId, + result_type: 'model_size_stats', + model_bytes: '1.8 MB', + model_bytes_exceeded: '0', + model_bytes_memory_limit: '20971520', + total_by_field_count: '59', + total_over_field_count: '0', + total_partition_field_count: '58', + bucket_allocation_failures_count: '0', + memory_status: 'ok', + timestamp: '2016-02-11 23:30:00', + }; + await ml.jobTable.assertJobRowDetailsCounts(jobId, expectedCounts, expectedModelSizeStats); + }); + }); +} diff --git a/x-pack/test/functional/apps/machine_learning/create_single_metric_job.ts b/x-pack/test/functional/apps/machine_learning/create_single_metric_job.ts index a8993f7c92c1..2e258b4164af 100644 --- a/x-pack/test/functional/apps/machine_learning/create_single_metric_job.ts +++ b/x-pack/test/functional/apps/machine_learning/create_single_metric_job.ts @@ -146,9 +146,7 @@ export default function({ getService }: FtrProviderContext) { }); it('displays details for the created job in the job list', async () => { - const rows = await ml.jobTable.parseJobTable(); - const job = rows.filter(row => row.id === jobId)[0]; - expect(job).to.eql({ + const expectedRow = { id: jobId, description: jobDescription, jobGroups, @@ -157,18 +155,10 @@ export default function({ getService }: FtrProviderContext) { jobState: 'closed', datafeedState: 'stopped', latestTimestamp: '2016-02-11 23:56:59', - }); - - const countDetails = await ml.jobTable.parseJobCounts(jobId); - const counts = countDetails.counts; - - // last_data_time holds a runtime timestamp and is hard to predict - // the property is only validated to be present and then removed - // so it doesn't make the counts object validation fail - expect(counts).to.have.property('last_data_time'); - delete counts.last_data_time; + }; + await ml.jobTable.assertJobRowFields(jobId, expectedRow); - expect(counts).to.eql({ + const expectedCounts = { job_id: jobId, processed_record_count: '2,399', processed_field_count: '4,798', @@ -184,17 +174,8 @@ export default function({ getService }: FtrProviderContext) { latest_record_timestamp: '2016-02-11 23:56:59', input_record_count: '2,399', latest_bucket_timestamp: '2016-02-11 23:30:00', - }); - - const modelSizeStats = countDetails.modelSizeStats; - - // log_time holds a runtime timestamp and is hard to predict - // the property is only validated to be present and then removed - // so it doesn't make the modelSizeStats object validation fail - expect(modelSizeStats).to.have.property('log_time'); - delete modelSizeStats.log_time; - - expect(modelSizeStats).to.eql({ + }; + const expectedModelSizeStats = { job_id: jobId, result_type: 'model_size_stats', model_bytes: '47.6 KB', @@ -206,7 +187,8 @@ export default function({ getService }: FtrProviderContext) { bucket_allocation_failures_count: '0', memory_status: 'ok', timestamp: '2016-02-11 23:00:00', - }); + }; + await ml.jobTable.assertJobRowDetailsCounts(jobId, expectedCounts, expectedModelSizeStats); }); }); } diff --git a/x-pack/test/functional/apps/machine_learning/index.ts b/x-pack/test/functional/apps/machine_learning/index.ts index 31f7d67717ae..42cf2f35ee43 100644 --- a/x-pack/test/functional/apps/machine_learning/index.ts +++ b/x-pack/test/functional/apps/machine_learning/index.ts @@ -12,5 +12,6 @@ export default function({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./feature_controls')); loadTestFile(require.resolve('./pages')); loadTestFile(require.resolve('./create_single_metric_job')); + loadTestFile(require.resolve('./create_multi_metric_job')); }); } diff --git a/x-pack/test/functional/services/machine_learning/job_table.ts b/x-pack/test/functional/services/machine_learning/job_table.ts index 8e9c74f7c22c..85142edb6507 100644 --- a/x-pack/test/functional/services/machine_learning/job_table.ts +++ b/x-pack/test/functional/services/machine_learning/job_table.ts @@ -3,6 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ +import expect from '@kbn/expect'; import { FtrProviderContext } from '../../ftr_provider_context'; @@ -174,5 +175,38 @@ export function MachineLearningJobTableProvider({ getService }: FtrProviderConte await searchBarInput.clearValueWithKeyboard(); await searchBarInput.type(filter); } + + public async assertJobRowFields(jobId: string, expectedRow: object) { + const rows = await this.parseJobTable(); + const jobRow = rows.filter(row => row.id === jobId)[0]; + expect(jobRow).to.eql(expectedRow); + } + + public async assertJobRowDetailsCounts( + jobId: string, + expectedCounts: object, + expectedModelSizeStats: object + ) { + const countDetails = await this.parseJobCounts(jobId); + const counts = countDetails.counts; + + // last_data_time holds a runtime timestamp and is hard to predict + // the property is only validated to be present and then removed + // so it doesn't make the counts object validation fail + expect(counts).to.have.property('last_data_time'); + delete counts.last_data_time; + + expect(counts).to.eql(expectedCounts); + + const modelSizeStats = countDetails.modelSizeStats; + + // log_time holds a runtime timestamp and is hard to predict + // the property is only validated to be present and then removed + // so it doesn't make the modelSizeStats object validation fail + expect(modelSizeStats).to.have.property('log_time'); + delete modelSizeStats.log_time; + + expect(modelSizeStats).to.eql(expectedModelSizeStats); + } })(); } diff --git a/x-pack/test/functional/services/machine_learning/job_type_selection.ts b/x-pack/test/functional/services/machine_learning/job_type_selection.ts index b6a6e613d075..c488c6bc5168 100644 --- a/x-pack/test/functional/services/machine_learning/job_type_selection.ts +++ b/x-pack/test/functional/services/machine_learning/job_type_selection.ts @@ -14,5 +14,10 @@ export function MachineLearningJobTypeSelectionProvider({ getService }: FtrProvi await testSubjects.clickWhenNotDisabled('mlJobTypeLinkSingleMetricJob'); await testSubjects.existOrFail('mlPageJobWizard'); }, + + async selectMultiMetricJob() { + await testSubjects.clickWhenNotDisabled('mlJobTypeLinkMultiMetricJob'); + await testSubjects.existOrFail('mlPageJobWizard'); + }, }; } diff --git a/x-pack/test/functional/services/machine_learning/job_wizard_common.ts b/x-pack/test/functional/services/machine_learning/job_wizard_common.ts index eea5e2a606d0..7a593d3a367c 100644 --- a/x-pack/test/functional/services/machine_learning/job_wizard_common.ts +++ b/x-pack/test/functional/services/machine_learning/job_wizard_common.ts @@ -89,7 +89,6 @@ export function MachineLearningJobWizardCommonProvider({ getService }: FtrProvid const comboBoxSelectedOptions = await comboBox.getComboBoxSelectedOptions( 'mlJobWizardComboBoxJobGroups comboBoxInput' ); - expect(comboBoxSelectedOptions.length).to.eql(jobGroups.length); expect(comboBoxSelectedOptions).to.eql(jobGroups); }, @@ -139,6 +138,60 @@ export function MachineLearningJobWizardCommonProvider({ getService }: FtrProvid expect(actualModelMemoryLimit).to.eql(expectedValue); }, + async assertMultiMetricSplitFieldInputExists() { + await testSubjects.existOrFail('mlJobWizardSplitFieldSelection comboBoxInput'); + }, + + async assertMultiMetricSplitFieldSelection(identifier: string) { + const comboBoxSelectedOptions = await comboBox.getComboBoxSelectedOptions( + 'mlJobWizardSplitFieldSelection comboBoxInput' + ); + expect(comboBoxSelectedOptions.length).to.eql(1); + expect(comboBoxSelectedOptions[0]).to.eql(identifier); + }, + + async assertInfluencerInputExists() { + await testSubjects.existOrFail('mlJobWizardInfluencerSelection comboBoxInput'); + }, + + async assertInfluencerSelection(influencers: string[]) { + const comboBoxSelectedOptions = await comboBox.getComboBoxSelectedOptions( + 'mlJobWizardInfluencerSelection comboBoxInput' + ); + expect(comboBoxSelectedOptions).to.eql(influencers); + }, + + async assertDetectorPreviewExists( + aggAndFieldIdentifier: string, + detectorPosition: number, + chartType: string + ) { + await testSubjects.existOrFail(`detector&${detectorPosition}`); + await testSubjects.existOrFail(`detector&${detectorPosition} detectorTitle`); + expect( + await testSubjects.getVisibleText(`detector&${detectorPosition} detectorTitle`) + ).to.eql(aggAndFieldIdentifier); + await testSubjects.existOrFail(`detector&${detectorPosition} mlAnomalyChart&${chartType}`); + }, + + async assertDetectorSplitExists(splitField: string) { + await testSubjects.existOrFail(`dataSplit dataSplitTitle&${splitField}`); + await testSubjects.existOrFail(`dataSplit splitCard&front`); + await testSubjects.existOrFail(`dataSplit splitCard&back`); + }, + + async assertDetectorSplitFrontCardTitle(frontCardTitle: string) { + expect(await testSubjects.getVisibleText(`dataSplit splitCard&front splitCardTitle`)).to.eql( + frontCardTitle + ); + }, + + async assertDetectorSplitNumberOfBackCards(numberOfBackCards: number) { + expect(await testSubjects.findAll(`dataSplit splitCard&back`)).to.have.length( + numberOfBackCards + ); + }, + async clickNextButton() { await testSubjects.clickWhenNotDisabled('mlJobWizardNavButtonNext'); }, @@ -171,6 +224,14 @@ export function MachineLearningJobWizardCommonProvider({ getService }: FtrProvid await comboBox.setCustom('mlJobWizardComboBoxJobGroups comboBoxInput', jobGroup); }, + async selectMultiMetricSplitField(identifier: string) { + await comboBox.set('mlJobWizardSplitFieldSelection comboBoxInput', identifier); + }, + + async addInfluencer(influencer: string) { + await comboBox.setCustom('mlJobWizardInfluencerSelection comboBoxInput', influencer); + }, + async ensureAdvancedSectionOpen() { await retry.try(async () => { if ((await testSubjects.exists('mlJobWizardAdvancedSection')) === false) {