Re-enable Firefox functional testing on CI (#64159)

* bring testing on Firefox back * disable some tests * skip more tests * cut more suites for Firefox * skip more tests for Firefox * replace smoke tag with includeFirefox Co-authored-by: Elastic Machine <[email protected]>
elastic · Apr 24, 2020 · c0bf910 · c0bf910
1 parent 085c245
commit c0bf910
Show file tree

Hide file tree

Showing 64 changed files with 39 additions and 74 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -11,7 +11,7 @@ kibanaPipeline(timeoutMinutes: 135, checkPrChanges: true) {
         'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'),
         'x-pack-intake-agent': workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh'),
         'kibana-oss-agent': workers.functional('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
-          // 'oss-firefoxSmoke': kibanaPipeline.functionalTestProcess('kibana-firefoxSmoke', './test/scripts/jenkins_firefox_smoke.sh'),
+          'oss-firefoxSmoke': kibanaPipeline.functionalTestProcess('kibana-firefoxSmoke', './test/scripts/jenkins_firefox_smoke.sh'),
           'oss-ciGroup1': kibanaPipeline.ossCiGroupProcess(1),
           'oss-ciGroup2': kibanaPipeline.ossCiGroupProcess(2),
           'oss-ciGroup3': kibanaPipeline.ossCiGroupProcess(3),
@@ -28,7 +28,7 @@ kibanaPipeline(timeoutMinutes: 135, checkPrChanges: true) {
           // 'oss-visualRegression': kibanaPipeline.functionalTestProcess('visualRegression', './test/scripts/jenkins_visual_regression.sh'),
         ]),
         'kibana-xpack-agent': workers.functional('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
-          // 'xpack-firefoxSmoke': kibanaPipeline.functionalTestProcess('xpack-firefoxSmoke', './test/scripts/jenkins_xpack_firefox_smoke.sh'),
+          'xpack-firefoxSmoke': kibanaPipeline.functionalTestProcess('xpack-firefoxSmoke', './test/scripts/jenkins_xpack_firefox_smoke.sh'),
           'xpack-ciGroup1': kibanaPipeline.xpackCiGroupProcess(1),
           'xpack-ciGroup2': kibanaPipeline.xpackCiGroupProcess(2),
           'xpack-ciGroup3': kibanaPipeline.xpackCiGroupProcess(3),

diff --git a/docs/developer/core/development-functional-tests.asciidoc b/docs/developer/core/development-functional-tests.asciidoc
@@ -154,16 +154,16 @@ A test suite is a collection of tests defined by calling `describe()`, and then
 Use tags in `describe()` function to group functional tests. Tags include:
 * `ciGroup{id}` - Assigns test suite to a specific CI worker
 * `skipCloud` and `skipFirefox` - Excludes test suite from running on Cloud or Firefox
-* `smoke` - Groups tests that run on Chrome and Firefox
+* `includeFirefox` - Groups tests that run on Chrome and Firefox
 
 **Cross-browser testing**:::
-On CI, all the functional tests are executed in Chrome by default. To also run a suite against Firefox, assign the `smoke` tag:
+On CI, all the functional tests are executed in Chrome by default. To also run a suite against Firefox, assign the `includeFirefox` tag:
 
 ["source","js"]
 -----------
 // on CI test suite will be run twice: in Chrome and Firefox
 describe('My Cross-browser Test Suite', function () {
-  this.tags('smoke');
+  this.tags('includeFirefox');
 
   it('My First Test');
 }

diff --git a/test/functional/apps/console/_console.ts b/test/functional/apps/console/_console.ts
@@ -40,7 +40,7 @@ export default function({ getService, getPageObjects }: FtrProviderContext) {
   const PageObjects = getPageObjects(['common', 'console']);
 
   describe('console app', function describeIndexTests() {
-    this.tags('smoke');
+    this.tags('includeFirefox');
     before(async () => {
       log.debug('navigateTo console');
       await PageObjects.common.navigateToApp('console');

diff --git a/test/functional/apps/context/_discover_navigation.js b/test/functional/apps/context/_discover_navigation.js
@@ -33,7 +33,6 @@ export default function({ getService, getPageObjects }) {
 
   // FLAKY: https://github.com/elastic/kibana/issues/53308
   describe.skip('context link in discover', function contextSize() {
-    this.tags('smoke');
     before(async function() {
       await PageObjects.common.navigateToApp('discover');
       await PageObjects.timePicker.setDefaultAbsoluteRange();

diff --git a/test/functional/apps/dashboard/dashboard_filtering.js b/test/functional/apps/dashboard/dashboard_filtering.js
@@ -38,7 +38,7 @@ export default function({ getService, getPageObjects }) {
   const PageObjects = getPageObjects(['common', 'dashboard', 'header', 'visualize', 'timePicker']);
 
   describe('dashboard filtering', function() {
-    this.tags('smoke');
+    this.tags('includeFirefox');
 
     before(async () => {
       await esArchiver.load('dashboard/current/kibana');

diff --git a/test/functional/apps/dashboard/dashboard_save.js b/test/functional/apps/dashboard/dashboard_save.js
@@ -24,7 +24,7 @@ export default function({ getPageObjects, getService }) {
   const listingTable = getService('listingTable');
 
   describe('dashboard save', function describeIndexTests() {
-    this.tags('smoke');
+    this.tags('includeFirefox');
     const dashboardName = 'Dashboard Save Test';
     const dashboardNameEnterKey = 'Dashboard Save Test with Enter Key';
 

diff --git a/test/functional/apps/dashboard/panel_controls.js b/test/functional/apps/dashboard/panel_controls.js
@@ -43,8 +43,6 @@ export default function({ getService, getPageObjects }) {
   const dashboardName = 'Dashboard Panel Controls Test';
 
   describe('dashboard panel controls', function viewEditModeTests() {
-    this.tags('smoke');
-
     before(async function() {
       await PageObjects.dashboard.initTests();
       await PageObjects.dashboard.preserveCrossAppState();

diff --git a/test/functional/apps/dashboard/time_zones.js b/test/functional/apps/dashboard/time_zones.js
@@ -27,7 +27,7 @@ export default function({ getService, getPageObjects }) {
   const PageObjects = getPageObjects(['dashboard', 'timePicker', 'settings', 'common']);
 
   describe('dashboard time zones', function() {
-    this.tags('smoke');
+    this.tags('includeFirefox');
 
     before(async () => {
       await esArchiver.load('dashboard/current/kibana');

diff --git a/test/functional/apps/discover/_doc_navigation.js b/test/functional/apps/discover/_doc_navigation.js
@@ -33,7 +33,6 @@ export default function({ getService, getPageObjects }) {
 
   // FLAKY: https://github.com/elastic/kibana/issues/62281
   describe.skip('doc link in discover', function contextSize() {
-    this.tags('smoke');
     before(async function() {
       await esArchiver.loadIfNeeded('logstash_functional');
       await PageObjects.common.navigateToApp('discover');

diff --git a/test/functional/apps/discover/_field_data.js b/test/functional/apps/discover/_field_data.js
@@ -28,7 +28,7 @@ export default function({ getService, getPageObjects }) {
   const PageObjects = getPageObjects(['common', 'header', 'discover', 'visualize', 'timePicker']);
 
   describe('discover tab', function describeIndexTests() {
-    this.tags('smoke');
+    this.tags('includeFirefox');
     before(async function() {
       await esArchiver.loadIfNeeded('logstash_functional');
       await esArchiver.load('discover');

diff --git a/test/functional/apps/getting_started/index.js b/test/functional/apps/getting_started/index.js
@@ -21,7 +21,7 @@ export default function({ getService, loadTestFile }) {
   const browser = getService('browser');
 
   describe('Getting Started ', function() {
-    this.tags(['ciGroup6', 'smoke']);
+    this.tags(['ciGroup6']);
 
     before(async function() {
       await browser.setWindowSize(1200, 800);

diff --git a/test/functional/apps/home/_home.js b/test/functional/apps/home/_home.js
@@ -25,7 +25,7 @@ export default function({ getService, getPageObjects }) {
   const PageObjects = getPageObjects(['common', 'home']);
 
   describe('Kibana takes you home', function describeIndexTests() {
-    this.tags('smoke');
+    this.tags('includeFirefox');
 
     it('clicking on kibana logo should take you to home page', async () => {
       await PageObjects.common.navigateToApp('settings');

diff --git a/test/functional/apps/home/_sample_data.ts b/test/functional/apps/home/_sample_data.ts
@@ -32,8 +32,6 @@ export default function({ getService, getPageObjects }: FtrProviderContext) {
   const PageObjects = getPageObjects(['common', 'header', 'home', 'dashboard', 'timePicker']);
 
   describe('sample data', function describeIndexTests() {
-    this.tags('smoke');
-
     before(async () => {
       await security.testUser.setRoles(['kibana_admin', 'kibana_sample_admin']);
       await PageObjects.common.navigateToUrl('home', 'tutorial_directory/sampleData');

diff --git a/test/functional/apps/management/_create_index_pattern_wizard.js b/test/functional/apps/management/_create_index_pattern_wizard.js
@@ -25,8 +25,6 @@ export default function({ getService, getPageObjects }) {
   const PageObjects = getPageObjects(['settings', 'common']);
 
   describe('"Create Index Pattern" wizard', function() {
-    this.tags('smoke');
-
     before(async function() {
       // delete .kibana index and then wait for Kibana to re-create it
       await kibanaServer.uiSettings.replace({});

diff --git a/test/functional/apps/visualize/_experimental_vis.js b/test/functional/apps/visualize/_experimental_vis.js
@@ -24,8 +24,6 @@ export default ({ getService, getPageObjects }) => {
   const PageObjects = getPageObjects(['visualize']);
 
   describe('experimental visualizations in visualize app ', function() {
-    this.tags('smoke');
-
     describe('experimental visualizations', () => {
       beforeEach(async () => {
         log.debug('navigateToApp visualize');

diff --git a/test/functional/apps/visualize/_gauge_chart.js b/test/functional/apps/visualize/_gauge_chart.js
@@ -28,8 +28,6 @@ export default function({ getService, getPageObjects }) {
 
   // FLAKY: https://github.com/elastic/kibana/issues/45089
   describe('gauge chart', function indexPatternCreation() {
-    this.tags('smoke');
-
     async function initGaugeVis() {
       log.debug('navigateToApp visualize');
       await PageObjects.visualize.navigateToNewVisualization();

diff --git a/test/functional/apps/visualize/_heatmap_chart.js b/test/functional/apps/visualize/_heatmap_chart.js
@@ -25,7 +25,6 @@ export default function({ getService, getPageObjects }) {
   const PageObjects = getPageObjects(['visualize', 'visEditor', 'visChart', 'timePicker']);
 
   describe('heatmap chart', function indexPatternCreation() {
-    this.tags('smoke');
     const vizName1 = 'Visualization HeatmapChart';
 
     before(async function() {

diff --git a/test/functional/apps/visualize/_inspector.js b/test/functional/apps/visualize/_inspector.js
@@ -24,7 +24,6 @@ export default function({ getService, getPageObjects }) {
   const PageObjects = getPageObjects(['visualize', 'visEditor', 'visChart', 'timePicker']);
 
   describe('inspector', function describeIndexTests() {
-    this.tags('smoke');
     before(async function() {
       await PageObjects.visualize.navigateToNewVisualization();
       await PageObjects.visualize.clickVerticalBarChart();

diff --git a/test/functional/apps/visualize/_tsvb_chart.ts b/test/functional/apps/visualize/_tsvb_chart.ts
@@ -29,7 +29,6 @@ export default function({ getService, getPageObjects }: FtrProviderContext) {
   const PageObjects = getPageObjects(['visualize', 'visualBuilder', 'timePicker', 'visChart']);
 
   describe('visual builder', function describeIndexTests() {
-    this.tags('smoke');
     beforeEach(async () => {
       await security.testUser.setRoles(['kibana_admin', 'test_logstash_reader']);
       await PageObjects.visualize.navigateToNewVisualization();

diff --git a/test/functional/apps/visualize/input_control_vis/chained_controls.js b/test/functional/apps/visualize/input_control_vis/chained_controls.js
@@ -27,7 +27,7 @@ export default function({ getService, getPageObjects }) {
   const comboBox = getService('comboBox');
 
   describe('chained controls', function() {
-    this.tags('smoke');
+    this.tags('includeFirefox');
 
     before(async () => {
       await PageObjects.common.navigateToApp('visualize');

diff --git a/test/scripts/jenkins_firefox_smoke.sh b/test/scripts/jenkins_firefox_smoke.sh
@@ -6,5 +6,5 @@ checks-reporter-with-killswitch "Firefox smoke test" \
   node scripts/functional_tests \
     --bail --debug \
     --kibana-install-dir "$installDir" \
-    --include-tag "smoke" \
+    --include-tag "includeFirefox" \
     --config test/functional/config.firefox.js;
diff --git a/test/scripts/jenkins_xpack_firefox_smoke.sh b/test/scripts/jenkins_xpack_firefox_smoke.sh
@@ -6,5 +6,5 @@ checks-reporter-with-killswitch "X-Pack firefox smoke test" \
   node scripts/functional_tests \
     --debug --bail \
     --kibana-install-dir "$KIBANA_INSTALL_DIR" \
-    --include-tag "smoke" \
+    --include-tag "includeFirefox" \
     --config test/functional/config.firefox.js;
diff --git a/x-pack/test/functional/apps/api_keys/home_page.ts b/x-pack/test/functional/apps/api_keys/home_page.ts
@@ -13,7 +13,6 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
   const security = getService('security');
 
   describe('Home page', function() {
-    this.tags('smoke');
     before(async () => {
       await security.testUser.setRoles(['kibana_admin']);
       await pageObjects.common.navigateToApp('apiKeys');

diff --git a/x-pack/test/functional/apps/canvas/smoke_test.js b/x-pack/test/functional/apps/canvas/smoke_test.js
@@ -15,7 +15,7 @@ export default function canvasSmokeTest({ getService, getPageObjects }) {
   const PageObjects = getPageObjects(['common']);
 
   describe('smoke test', function() {
-    this.tags('smoke');
+    this.tags('includeFirefox');
     const workpadListSelector = 'canvasWorkpadLoaderTable > canvasWorkpadLoaderWorkpad';
     const testWorkpadId = 'workpad-1705f884-6224-47de-ba49-ca224fe6ec31';
 

diff --git a/x-pack/test/functional/apps/cross_cluster_replication/home_page.ts b/x-pack/test/functional/apps/cross_cluster_replication/home_page.ts
@@ -12,7 +12,6 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
   const log = getService('log');
 
   describe('Home page', function() {
-    this.tags('smoke');
     before(async () => {
       await pageObjects.common.navigateToApp('crossClusterReplication');
     });

diff --git a/x-pack/test/functional/apps/grok_debugger/grok_debugger.js b/x-pack/test/functional/apps/grok_debugger/grok_debugger.js
@@ -12,7 +12,7 @@ export default function({ getService, getPageObjects }) {
   const PageObjects = getPageObjects(['grokDebugger']);
 
   describe('grok debugger app', function() {
-    this.tags('smoke');
+    this.tags('includeFirefox');
     before(async () => {
       await esArchiver.load('empty_kibana');
       // Increase window height to ensure "Simulate" button is shown above the

diff --git a/x-pack/test/functional/apps/index_lifecycle_management/home_page.ts b/x-pack/test/functional/apps/index_lifecycle_management/home_page.ts
@@ -12,7 +12,6 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
   const log = getService('log');
 
   describe('Home page', function() {
-    this.tags('smoke');
     before(async () => {
       await pageObjects.common.navigateToApp('indexLifecycleManagement');
     });

diff --git a/x-pack/test/functional/apps/index_management/home_page.ts b/x-pack/test/functional/apps/index_management/home_page.ts
@@ -14,7 +14,6 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
   const browser = getService('browser');
 
   describe('Home page', function() {
-    this.tags('smoke');
     before(async () => {
       await pageObjects.common.navigateToApp('indexManagement');
     });

diff --git a/x-pack/test/functional/apps/infra/home_page.ts b/x-pack/test/functional/apps/infra/home_page.ts
@@ -15,7 +15,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
   const pageObjects = getPageObjects(['common', 'infraHome']);
 
   describe('Home page', function() {
-    this.tags('smoke');
+    this.tags('includeFirefox');
     before(async () => {
       await esArchiver.load('empty_kibana');
     });

diff --git a/x-pack/test/functional/apps/infra/link_to.ts b/x-pack/test/functional/apps/infra/link_to.ts
@@ -21,7 +21,6 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
   const traceId = '433b4651687e18be2c6c8e3b11f53d09';
 
   describe('Infra link-to', function() {
-    this.tags('smoke');
     it('redirects to the logs app and parses URL search params correctly', async () => {
       const location = {
         hash: '',

diff --git a/x-pack/test/functional/apps/infra/log_entry_categories_tab.ts b/x-pack/test/functional/apps/infra/log_entry_categories_tab.ts
@@ -13,7 +13,7 @@ export default ({ getService }: FtrProviderContext) => {
   const retry = getService('retry');
 
   describe('Log Entry Categories Tab', function() {
-    this.tags('smoke');
+    this.tags('includeFirefox');
 
     describe('with a trial license', () => {
       it('is visible', async () => {

diff --git a/x-pack/test/functional/apps/infra/log_entry_rate_tab.ts b/x-pack/test/functional/apps/infra/log_entry_rate_tab.ts
@@ -13,7 +13,7 @@ export default ({ getService }: FtrProviderContext) => {
   const retry = getService('retry');
 
   describe('Log Entry Rate Tab', function() {
-    this.tags('smoke');
+    this.tags('includeFirefox');
 
     describe('with a trial license', () => {
       it('is visible', async () => {

diff --git a/x-pack/test/functional/apps/infra/logs_source_configuration.ts b/x-pack/test/functional/apps/infra/logs_source_configuration.ts
@@ -17,8 +17,6 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
   const retry = getService('retry');
 
   describe('Logs Source Configuration', function() {
-    this.tags('smoke');
-
     before(async () => {
       await esArchiver.load('empty_kibana');
     });

diff --git a/x-pack/test/functional/apps/infra/metrics_source_configuration.ts b/x-pack/test/functional/apps/infra/metrics_source_configuration.ts
@@ -15,7 +15,6 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
   const pageObjects = getPageObjects(['common', 'infraHome']);
 
   describe('Infrastructure Source Configuration', function() {
-    this.tags('smoke');
     before(async () => {
       await esArchiver.load('empty_kibana');
     });

diff --git a/x-pack/test/functional/apps/logstash/index.js b/x-pack/test/functional/apps/logstash/index.js
@@ -6,7 +6,7 @@
 
 export default function({ loadTestFile }) {
   describe('logstash', function() {
-    this.tags(['ciGroup2', 'smoke']);
+    this.tags(['ciGroup2']);
 
     loadTestFile(require.resolve('./pipeline_list'));
     loadTestFile(require.resolve('./pipeline_create'));

diff --git a/x-pack/test/functional/apps/machine_learning/anomaly_detection/advanced_job.ts b/x-pack/test/functional/apps/machine_learning/anomaly_detection/advanced_job.ts
@@ -277,7 +277,7 @@ export default function({ getService }: FtrProviderContext) {
   const calendarId = `wizard-test-calendar_${Date.now()}`;
 
   describe('advanced job', function() {
-    this.tags(['smoke', 'mlqa']);
+    this.tags(['mlqa']);
     before(async () => {
       await esArchiver.loadIfNeeded('ml/ecommerce');
       await ml.testResources.createIndexPatternIfNeeded('ft_ecommerce', 'order_date');

diff --git a/x-pack/test/functional/apps/machine_learning/anomaly_detection/anomaly_explorer.ts b/x-pack/test/functional/apps/machine_learning/anomaly_detection/anomaly_explorer.ts
@@ -57,7 +57,7 @@ export default function({ getService }: FtrProviderContext) {
   const ml = getService('ml');
 
   describe('anomaly explorer', function() {
-    this.tags(['smoke', 'mlqa']);
+    this.tags(['mlqa']);
     before(async () => {
       await esArchiver.loadIfNeeded('ml/farequote');
       await ml.testResources.createIndexPatternIfNeeded('ft_farequote', '@timestamp');

diff --git a/x-pack/test/functional/apps/machine_learning/anomaly_detection/categorization_job.ts b/x-pack/test/functional/apps/machine_learning/anomaly_detection/categorization_job.ts
@@ -77,7 +77,7 @@ export default function({ getService }: FtrProviderContext) {
   const calendarId = `wizard-test-calendar_${Date.now()}`;
 
   describe('categorization', function() {
-    this.tags(['smoke', 'mlqa']);
+    this.tags(['mlqa']);
     before(async () => {
       await esArchiver.loadIfNeeded('ml/categorization');
       await ml.testResources.createIndexPatternIfNeeded('ft_categorization', '@timestamp');

diff --git a/x-pack/test/functional/apps/machine_learning/anomaly_detection/date_nanos_job.ts b/x-pack/test/functional/apps/machine_learning/anomaly_detection/date_nanos_job.ts
@@ -166,7 +166,7 @@ export default function({ getService }: FtrProviderContext) {
   ];
 
   describe('job on data set with date_nanos time field', function() {
-    this.tags(['smoke', 'mlqa']);
+    this.tags(['mlqa']);
     before(async () => {
       await esArchiver.loadIfNeeded('ml/event_rate_nanos');
       await ml.testResources.createIndexPatternIfNeeded(

diff --git a/x-pack/test/functional/apps/machine_learning/anomaly_detection/multi_metric_job.ts b/x-pack/test/functional/apps/machine_learning/anomaly_detection/multi_metric_job.ts
@@ -74,7 +74,7 @@ export default function({ getService }: FtrProviderContext) {
   const calendarId = `wizard-test-calendar_${Date.now()}`;
 
   describe('multi metric', function() {
-    this.tags(['smoke', 'mlqa']);
+    this.tags(['mlqa']);
     before(async () => {
       await esArchiver.loadIfNeeded('ml/farequote');
       await ml.testResources.createIndexPatternIfNeeded('ft_farequote', '@timestamp');

diff --git a/x-pack/test/functional/apps/machine_learning/anomaly_detection/population_job.ts b/x-pack/test/functional/apps/machine_learning/anomaly_detection/population_job.ts
@@ -88,7 +88,7 @@ export default function({ getService }: FtrProviderContext) {
   const calendarId = `wizard-test-calendar_${Date.now()}`;
 
   describe('population', function() {
-    this.tags(['smoke', 'mlqa']);
+    this.tags(['mlqa']);
     before(async () => {
       await esArchiver.loadIfNeeded('ml/ecommerce');
       await ml.testResources.createIndexPatternIfNeeded('ft_ecommerce', 'order_date');

diff --git a/x-pack/test/functional/apps/machine_learning/anomaly_detection/saved_search_job.ts b/x-pack/test/functional/apps/machine_learning/anomaly_detection/saved_search_job.ts
@@ -272,7 +272,7 @@ export default function({ getService }: FtrProviderContext) {
   ];
 
   describe('saved search', function() {
-    this.tags(['smoke', 'mlqa']);
+    this.tags(['mlqa']);
     before(async () => {
       await esArchiver.loadIfNeeded('ml/farequote');
       await ml.testResources.createIndexPatternIfNeeded('ft_farequote', '@timestamp');

diff --git a/x-pack/test/functional/apps/machine_learning/anomaly_detection/single_metric_job.ts b/x-pack/test/functional/apps/machine_learning/anomaly_detection/single_metric_job.ts
@@ -73,7 +73,7 @@ export default function({ getService }: FtrProviderContext) {
   const calendarId = `wizard-test-calendar_${Date.now()}`;
 
   describe('single metric', function() {
-    this.tags(['smoke', 'mlqa']);
+    this.tags(['mlqa']);
     before(async () => {
       await esArchiver.loadIfNeeded('ml/farequote');
       await ml.testResources.createIndexPatternIfNeeded('ft_farequote', '@timestamp');