[ML] Functional Tests: Extend canvas assertion options. (#91473)

Updates the canvas element assertion service and stabilizes tests.
elastic · Mar 19, 2021 · 778bb69 · 778bb69
1 parent fc3125a
commit 778bb69
Show file tree

Hide file tree

Showing 14 changed files with 321 additions and 172 deletions.
diff --git a/x-pack/plugins/ml/public/application/_hacks.scss b/x-pack/plugins/ml/public/application/_hacks.scss
@@ -1,6 +1,6 @@
 .tab-datavisualizer_index_select,
 .tab-timeseriesexplorer,
-.tab-explorer, {
+.tab-explorer {
   // Make all page background white until More of the pages use EuiPage to wrap in panel-like components
   background-color: $euiColorEmptyShade;
 }
@@ -22,3 +22,12 @@
 .clear, .clearfix {
   clear: both;
 }
+
+// Helper class for functional tests to disable anti-aliasing for canvas elements
+.mlDisableAntiAliasing {
+  -webkit-font-smoothing : none;
+
+  * canvas {
+    image-rendering: pixelated;
+  }
+}
diff --git a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix.tsx b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix.tsx
@@ -267,7 +267,7 @@ export const ScatterplotMatrix: FC<ScatterplotMatrixProps> = ({
       {splom === undefined || vegaSpec === undefined ? (
         <VegaChartLoading />
       ) : (
-        <div data-test-subj="mlScatterplotMatrix">
+        <div data-test-subj={`mlScatterplotMatrix ${isLoading ? 'loading' : 'loaded'}`}>
           <EuiFlexGroup>
             <EuiFlexItem>
               <EuiFormRow

diff --git a/...lytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx b/...lytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx
@@ -120,14 +120,6 @@ export const ConfigurationStepForm: FC<CreateAnalyticsStepProps> = ({
     language: SEARCH_QUERY_LANGUAGE.KUERY,
   });
 
-  const scatterplotFieldOptions = useMemo(
-    () =>
-      includesTableItems
-        .filter((d) => d.feature_type === 'numerical' && d.is_included)
-        .map((d) => d.name),
-    [includesTableItems]
-  );
-
   const toastNotifications = getToastNotifications();
 
   const setJobConfigQuery: ExplorationQueryBarProps['setSearchQuery'] = (update) => {
@@ -341,16 +333,37 @@ export const ConfigurationStepForm: FC<CreateAnalyticsStepProps> = ({
     [currentIndexPattern.fields]
   );
 
+  const scatterplotMatrixProps = useMemo(
+    () => ({
+      color: isJobTypeWithDepVar ? dependentVariable : undefined,
+      fields: includesTableItems
+        .filter((d) => d.feature_type === 'numerical' && d.is_included)
+        .map((d) => d.name),
+      index: currentIndexPattern.title,
+      legendType: getScatterplotMatrixLegendType(jobType),
+      searchQuery: jobConfigQuery,
+    }),
+    [
+      currentIndexPattern.title,
+      dependentVariable,
+      includesTableItems,
+      isJobTypeWithDepVar,
+      jobConfigQuery,
+      jobType,
+    ]
+  );
+
   // Show the Scatterplot Matrix only if
   // - There's more than one suitable field available
   // - The job type is outlier detection, or
   // - The job type is regression or classification and the dependent variable has been set
-  const showScatterplotMatrix =
-    (jobType === ANALYSIS_CONFIG_TYPE.OUTLIER_DETECTION ||
-      ((jobType === ANALYSIS_CONFIG_TYPE.REGRESSION ||
-        jobType === ANALYSIS_CONFIG_TYPE.CLASSIFICATION) &&
-        !dependentVariableEmpty)) &&
-    scatterplotFieldOptions.length > 1;
+  const showScatterplotMatrix = useMemo(
+    () =>
+      (jobType === ANALYSIS_CONFIG_TYPE.OUTLIER_DETECTION ||
+        (isJobTypeWithDepVar && !dependentVariableEmpty)) &&
+      scatterplotMatrixProps.fields.length > 1,
+    [dependentVariableEmpty, jobType, scatterplotMatrixProps.fields.length]
+  );
 
   // Don't render until `savedSearchQuery` has been initialized.
   // `undefined` means uninitialized, `null` means initialized but not used.
@@ -550,18 +563,7 @@ export const ConfigurationStepForm: FC<CreateAnalyticsStepProps> = ({
             paddingSize="m"
             data-test-subj="mlAnalyticsCreateJobWizardScatterplotMatrixPanel"
           >
-            <ScatterplotMatrix
-              fields={scatterplotFieldOptions}
-              index={currentIndexPattern.title}
-              color={
-                jobType === ANALYSIS_CONFIG_TYPE.REGRESSION ||
-                jobType === ANALYSIS_CONFIG_TYPE.CLASSIFICATION
-                  ? dependentVariable
-                  : undefined
-              }
-              legendType={getScatterplotMatrixLegendType(jobType)}
-              searchQuery={jobConfigQuery}
-            />
+            <ScatterplotMatrix {...scatterplotMatrixProps} />
           </EuiPanel>
           <EuiSpacer />
         </>

diff --git a/..._exploration/components/classification_exploration/get_roc_curve_chart_vega_lite_spec.tsx b/..._exploration/components/classification_exploration/get_roc_curve_chart_vega_lite_spec.tsx
@@ -53,6 +53,7 @@ export const getRocCurveChartVegaLiteSpec = (
 
   return {
     $schema: 'https://vega.github.io/schema/vega-lite/v4.8.1.json',
+    background: 'transparent',
     // Left padding of 45px to align the left axis of the chart with the confusion matrix above.
     padding: { left: 45, top: 0, right: 0, bottom: 0 },
     config: {

diff --git a/x-pack/test/functional/apps/ml/data_frame_analytics/classification_creation.ts b/x-pack/test/functional/apps/ml/data_frame_analytics/classification_creation.ts
@@ -12,8 +12,7 @@ export default function ({ getService }: FtrProviderContext) {
   const ml = getService('ml');
   const editedDescription = 'Edited description';
 
-  // Failing: See https://github.com/elastic/kibana/issues/91450
-  describe.skip('classification creation', function () {
+  describe('classification creation', function () {
     before(async () => {
       await esArchiver.loadIfNeeded('ml/bm_classification');
       await ml.testResources.createIndexPatternIfNeeded('ft_bank_marketing', '@timestamp');
@@ -43,24 +42,21 @@ export default function ({ getService }: FtrProviderContext) {
         createIndexPattern: true,
         expected: {
           rocCurveColorState: [
-            // background
-            { key: '#FFFFFF', value: 93 },
             // tick/grid/axis
-            { key: '#98A2B3', value: 1 },
-            { key: '#DDDDDD', value: 3 },
-            // line
-            { key: '#6092C0', value: 1 },
+            { key: '#DDDDDD', value: 50 },
+            // lines
+            { key: '#98A2B3', value: 30 },
+            { key: '#6092C0', value: 10 },
+            { key: '#5F92C0', value: 6 },
           ],
           scatterplotMatrixColorStats: [
-            // background
-            { key: '#000000', value: 94 },
+            // marker colors
+            { key: '#7FC6B3', value: 1 },
+            { key: '#88ADD0', value: 0.03 },
             // tick/grid/axis
-            { key: '#DDDDDD', value: 1 },
-            { key: '#D3DAE6', value: 1 },
-            { key: '#F5F7FA', value: 1 },
-            // scatterplot circles
-            { key: '#6A717D', value: 1 },
-            { key: '#54B39A', value: 1 },
+            { key: '#DDDDDD', value: 8 },
+            { key: '#D3DAE6', value: 8 },
+            { key: '#F5F7FA', value: 20 },
           ],
           row: {
             type: 'classification',
@@ -112,8 +108,7 @@ export default function ({ getService }: FtrProviderContext) {
           await ml.dataFrameAnalyticsCreation.assertIncludeFieldsSelectionExists();
 
           await ml.testExecution.logTestStep('displays the scatterplot matrix');
-          await ml.dataFrameAnalyticsCanvasElement.assertCanvasElement(
-            'mlAnalyticsCreateJobWizardScatterplotMatrixFormRow',
+          await ml.dataFrameAnalyticsCreation.assertScatterplotMatrix(
             testData.expected.scatterplotMatrixColorStats
           );
 
@@ -157,7 +152,12 @@ export default function ({ getService }: FtrProviderContext) {
 
           await ml.testExecution.logTestStep('checks validation callouts exist');
           await ml.dataFrameAnalyticsCreation.assertValidationCalloutsExists();
-          await ml.dataFrameAnalyticsCreation.assertAllValidationCalloutsPresent(3);
+          // Expect the follow callouts:
+          // - ✓ Dependent variable
+          // - ✓ Training percent
+          // - ✓ Top classes
+          // - ⚠ Analysis fields
+          await ml.dataFrameAnalyticsCreation.assertAllValidationCalloutsPresent(4);
 
           await ml.testExecution.logTestStep('continues to the create step');
           await ml.dataFrameAnalyticsCreation.continueToCreateStep();
@@ -238,16 +238,21 @@ export default function ({ getService }: FtrProviderContext) {
           await ml.testExecution.logTestStep('displays the results view for created job');
           await ml.dataFrameAnalyticsTable.openResultsView(testData.jobId);
           await ml.dataFrameAnalyticsResults.assertClassificationEvaluatePanelElementsExists();
-          await ml.dataFrameAnalyticsCanvasElement.assertCanvasElement(
+          await ml.commonUI.assertColorsInCanvasElement(
             'mlDFAnalyticsClassificationExplorationRocCurveChart',
-            testData.expected.rocCurveColorState
+            testData.expected.rocCurveColorState,
+            ['#000000'],
+            undefined,
+            undefined,
+            // increased tolerance for ROC curve chart up from 10 to 20
+            // since the returned colors vary quite a bit on each run.
+            20
           );
           await ml.dataFrameAnalyticsResults.assertClassificationTablePanelExists();
           await ml.dataFrameAnalyticsResults.assertResultsTableExists();
           await ml.dataFrameAnalyticsResults.assertResultsTableTrainingFiltersExist();
           await ml.dataFrameAnalyticsResults.assertResultsTableNotEmpty();
-          await ml.dataFrameAnalyticsCanvasElement.assertCanvasElement(
-            'mlDFExpandableSection-splom',
+          await ml.dataFrameAnalyticsResults.assertScatterplotMatrix(
             testData.expected.scatterplotMatrixColorStats
           );
         });

diff --git a/x-pack/test/functional/apps/ml/data_frame_analytics/outlier_detection_creation.ts b/x-pack/test/functional/apps/ml/data_frame_analytics/outlier_detection_creation.ts
@@ -12,8 +12,7 @@ export default function ({ getService }: FtrProviderContext) {
   const ml = getService('ml');
   const editedDescription = 'Edited description';
 
-  // FAILING: https://github.com/elastic/kibana/issues/94854
-  describe.skip('outlier detection creation', function () {
+  describe('outlier detection creation', function () {
     before(async () => {
       await esArchiver.loadIfNeeded('ml/ihp_outlier');
       await ml.testResources.createIndexPatternIfNeeded('ft_ihp_outlier', '@timestamp');
@@ -51,26 +50,21 @@ export default function ({ getService }: FtrProviderContext) {
             { chartAvailable: true, id: 'Exterior2nd', legend: '3 categories' },
             { chartAvailable: true, id: 'Fireplaces', legend: '0 - 3' },
           ],
-          scatterplotMatrixColorStatsWizard: [
-            // background
-            { key: '#000000', value: 91 },
-            // tick/grid/axis
-            { key: '#6A717D', value: 2 },
-            { key: '#F5F7FA', value: 2 },
-            { key: '#D3DAE6', value: 1 },
-            // scatterplot circles
-            { key: '#69707D', value: 1 },
-            { key: '#98A1B3', value: 1 },
+          scatterplotMatrixColorsWizard: [
+            // markers
+            { key: '#52B398', value: 25 },
+            // grey boilerplate
+            { key: '#6A717D', value: 30 },
           ],
           scatterplotMatrixColorStatsResults: [
-            // background
-            { key: '#000000', value: 91 },
+            // red markers
+            { key: '#D98071', value: 1 },
             // tick/grid/axis, grey markers
-            // the red outlier color is not above the 1% threshold.
-            { key: '#6A717D', value: 2 },
-            { key: '#98A2B3', value: 1 },
-            { key: '#F5F7FA', value: 2 },
-            { key: '#D3DAE6', value: 1 },
+            { key: '#6A717D', value: 30 },
+            { key: '#D3DAE6', value: 8 },
+            { key: '#98A1B3', value: 25 },
+            // anti-aliasing
+            { key: '#F5F7FA', value: 27 },
           ],
           row: {
             type: 'outlier_detection',
@@ -94,6 +88,10 @@ export default function ({ getService }: FtrProviderContext) {
           await ml.navigation.navigateToDataFrameAnalytics();
 
           await ml.testExecution.logTestStep('loads the source selection modal');
+
+          // Disable anti-aliasing to stabilize canvas image rendering assertions
+          await ml.commonUI.disableAntiAliasing();
+
           await ml.dataFrameAnalytics.startAnalyticsCreation();
 
           await ml.testExecution.logTestStep(
@@ -129,9 +127,8 @@ export default function ({ getService }: FtrProviderContext) {
           await ml.dataFrameAnalyticsCreation.assertIncludeFieldsSelectionExists();
 
           await ml.testExecution.logTestStep('displays the scatterplot matrix');
-          await ml.dataFrameAnalyticsCanvasElement.assertCanvasElement(
-            'mlAnalyticsCreateJobWizardScatterplotMatrixFormRow',
-            testData.expected.scatterplotMatrixColorStatsWizard
+          await ml.dataFrameAnalyticsCreation.assertScatterplotMatrix(
+            testData.expected.scatterplotMatrixColorsWizard
           );
 
           await ml.testExecution.logTestStep('continues to the additional options step');
@@ -258,8 +255,7 @@ export default function ({ getService }: FtrProviderContext) {
           await ml.dataFrameAnalyticsResults.assertResultsTableExists();
           await ml.dataFrameAnalyticsResults.assertResultsTableNotEmpty();
           await ml.dataFrameAnalyticsResults.assertFeatureInfluenceCellNotEmpty();
-          await ml.dataFrameAnalyticsCanvasElement.assertCanvasElement(
-            'mlDFExpandableSection-splom',
+          await ml.dataFrameAnalyticsResults.assertScatterplotMatrix(
             testData.expected.scatterplotMatrixColorStatsResults
           );
         });

diff --git a/x-pack/test/functional/apps/ml/data_frame_analytics/regression_creation.ts b/x-pack/test/functional/apps/ml/data_frame_analytics/regression_creation.ts
@@ -41,14 +41,13 @@ export default function ({ getService }: FtrProviderContext) {
         createIndexPattern: true,
         expected: {
           scatterplotMatrixColorStats: [
-            // background
-            { key: '#000000', value: 80 },
+            // some marker colors of the continuous color scale
+            { key: '#61AFA3', value: 2 },
+            { key: '#D1E5E0', value: 2 },
             // tick/grid/axis
-            { key: '#6A717D', value: 1 },
-            { key: '#F5F7FA', value: 2 },
-            { key: '#D3DAE6', value: 1 },
-            // because a continuous color scale is used for the scatterplot circles,
-            // none of the generated colors is above the 1% threshold.
+            { key: '#6A717D', value: 10 },
+            { key: '#F5F7FA', value: 12 },
+            { key: '#D3DAE6', value: 3 },
           ],
           row: {
             type: 'regression',
@@ -101,8 +100,7 @@ export default function ({ getService }: FtrProviderContext) {
           await ml.dataFrameAnalyticsCreation.assertIncludeFieldsSelectionExists();
 
           await ml.testExecution.logTestStep('displays the scatterplot matrix');
-          await ml.dataFrameAnalyticsCanvasElement.assertCanvasElement(
-            'mlAnalyticsCreateJobWizardScatterplotMatrixFormRow',
+          await ml.dataFrameAnalyticsCreation.assertScatterplotMatrix(
             testData.expected.scatterplotMatrixColorStats
           );
 
@@ -231,8 +229,7 @@ export default function ({ getService }: FtrProviderContext) {
           await ml.dataFrameAnalyticsResults.assertResultsTableExists();
           await ml.dataFrameAnalyticsResults.assertResultsTableTrainingFiltersExist();
           await ml.dataFrameAnalyticsResults.assertResultsTableNotEmpty();
-          await ml.dataFrameAnalyticsCanvasElement.assertCanvasElement(
-            'mlDFExpandableSection-splom',
+          await ml.dataFrameAnalyticsResults.assertScatterplotMatrix(
             testData.expected.scatterplotMatrixColorStats
           );
         });