From 384cf7864b302e2023ff4e0243fbbdd48536b0e7 Mon Sep 17 00:00:00 2001 From: Achyut Jhunjhunwala Date: Wed, 28 Jun 2023 10:29:46 +0200 Subject: [PATCH] [APM] Revert skip for flaky Tests (#160391) ## Summary Closes - https://github.com/elastic/kibana/issues/120056 - https://github.com/elastic/kibana/issues/122001 - https://github.com/elastic/kibana/issues/127431 - https://github.com/elastic/kibana/issues/127416 ## Flakiness Testing Changes in the PR were tested with the flaky test runner, running 50 times for all 5 configs for APM 1. 4 config (Trial, Basic, Cloud and Rules) present inside `apm_api_integration` folder and 2. 1 config present inside `functional/apps/apm` https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/2499 --- test/functional/page_objects/time_picker.ts | 5 ----- x-pack/plugins/apm/dev_docs/testing.md | 4 ++-- .../tests/alerts/anomaly_alert.spec.ts | 3 +-- x-pack/test/apm_api_integration/tests/index.ts | 3 +-- .../__snapshots__/instance_details.spec.snap | 5 ++++- .../tests/service_overview/instance_details.spec.ts | 7 +++---- .../apm/correlations/failed_transaction_correlations.ts | 5 ++--- .../apps/apm/correlations/latency_correlations.ts | 8 ++++---- .../functional/apps/apm/feature_controls/apm_security.ts | 3 +-- 9 files changed, 18 insertions(+), 25 deletions(-) diff --git a/test/functional/page_objects/time_picker.ts b/test/functional/page_objects/time_picker.ts index 51d2f21290431..be43b0d2b29ec 100644 --- a/test/functional/page_objects/time_picker.ts +++ b/test/functional/page_objects/time_picker.ts @@ -144,12 +144,10 @@ export class TimePickerPageObject extends FtrService { } this.log.debug(`Setting absolute range to ${fromTime} to ${toTime}`); await this.showStartEndTimes(); - let panel!: WebElementWrapper; // set to time await this.retry.waitFor(`endDate is set to ${toTime}`, async () => { await this.testSubjects.click('superDatePickerendDatePopoverButton'); - panel = await this.getTimePickerPanel(); await this.testSubjects.click('superDatePickerAbsoluteTab'); await this.testSubjects.click('superDatePickerAbsoluteDateInput'); await this.inputValue('superDatePickerAbsoluteDateInput', toTime); @@ -164,8 +162,6 @@ export class TimePickerPageObject extends FtrService { // set from time await this.retry.waitFor(`startDate is set to ${fromTime}`, async () => { await this.testSubjects.click('superDatePickerstartDatePopoverButton'); - await this.waitPanelIsGone(panel); - panel = await this.getTimePickerPanel(); await this.testSubjects.click('superDatePickerAbsoluteTab'); await this.testSubjects.click('superDatePickerAbsoluteDateInput'); await this.inputValue('superDatePickerAbsoluteDateInput', fromTime); @@ -196,7 +192,6 @@ export class TimePickerPageObject extends FtrService { await this.testSubjects.click('querySubmitButton'); } - await this.waitPanelIsGone(panel); await this.header.awaitGlobalLoadingIndicatorHidden(); } diff --git a/x-pack/plugins/apm/dev_docs/testing.md b/x-pack/plugins/apm/dev_docs/testing.md index e18ef18a2b86a..533ad5f474fd8 100644 --- a/x-pack/plugins/apm/dev_docs/testing.md +++ b/x-pack/plugins/apm/dev_docs/testing.md @@ -116,13 +116,13 @@ TODO: We could try moving this tests to the new e2e tests located at `x-pack/plu **Start server** ``` -node scripts/functional_tests_server --config x-pack/test/functional/config.base.js +node scripts/functional_tests_server --config x-pack/test/functional/apps/apm/config.ts ``` **Run tests** ``` -node scripts/functional_test_runner --config x-pack/test/functional/config.base.js --grep='APM specs' +node scripts/functional_test_runner --config x-pack/test/functional/apps/apm/config.ts --grep='APM specs' ``` APM tests are located in `x-pack/test/functional/apps/apm`. diff --git a/x-pack/test/apm_api_integration/tests/alerts/anomaly_alert.spec.ts b/x-pack/test/apm_api_integration/tests/alerts/anomaly_alert.spec.ts index d5a4f9fe27462..e3f2a25af3d3d 100644 --- a/x-pack/test/apm_api_integration/tests/alerts/anomaly_alert.spec.ts +++ b/x-pack/test/apm_api_integration/tests/alerts/anomaly_alert.spec.ts @@ -23,8 +23,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { const es = getService('es'); const synthtraceEsClient = getService('synthtraceEsClient'); - - // FAILING VERSION BUMP: https://github.com/elastic/kibana/issues/155930 + // FLAKY https://github.com/elastic/kibana/issues/160298 registry.when.skip( 'fetching service anomalies with a trial license', { config: 'trial', archives: [] }, diff --git a/x-pack/test/apm_api_integration/tests/index.ts b/x-pack/test/apm_api_integration/tests/index.ts index 9de18c51500e3..aa3d8ac8afaeb 100644 --- a/x-pack/test/apm_api_integration/tests/index.ts +++ b/x-pack/test/apm_api_integration/tests/index.ts @@ -24,8 +24,7 @@ function getGlobPattern() { export default function apmApiIntegrationTests({ getService, loadTestFile }: FtrProviderContext) { const registry = getService('registry'); - // FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/160298 - describe.skip('APM API tests', function () { + describe('APM API tests', function () { const filePattern = getGlobPattern(); const tests = globby.sync(filePattern, { cwd }); diff --git a/x-pack/test/apm_api_integration/tests/service_overview/__snapshots__/instance_details.spec.snap b/x-pack/test/apm_api_integration/tests/service_overview/__snapshots__/instance_details.spec.snap index 357b6111da67d..fc1d1933056a3 100644 --- a/x-pack/test/apm_api_integration/tests/service_overview/__snapshots__/instance_details.spec.snap +++ b/x-pack/test/apm_api_integration/tests/service_overview/__snapshots__/instance_details.spec.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`APM API tests basic apm_8.0.0 Instance details when data is loaded fetch instance details return the correct data 1`] = ` +exports[`APM API tests service_overview/instance_details.spec.ts basic apm_8.0.0 Instance details when data is loaded fetch instance details return the correct data 1`] = ` Object { "agent": Object { "ephemeral_id": "2745d454-f57f-4473-a09b-fe6bef295860", @@ -34,10 +34,13 @@ Object { }, }, "kubernetes": Object { + "container": Object {}, + "deployment": Object {}, "pod": Object { "name": "opbeans-java-b8fcdf794-czvtr", "uid": "e0bf3100-4a0d-4d96-88ef-4e3aa3c78a5d", }, + "replicaset": Object {}, }, "service": Object { "environment": "production", diff --git a/x-pack/test/apm_api_integration/tests/service_overview/instance_details.spec.ts b/x-pack/test/apm_api_integration/tests/service_overview/instance_details.spec.ts index 7f4c0d643bf10..f6e257931ba54 100644 --- a/x-pack/test/apm_api_integration/tests/service_overview/instance_details.spec.ts +++ b/x-pack/test/apm_api_integration/tests/service_overview/instance_details.spec.ts @@ -46,12 +46,11 @@ export default function ApiTest({ getService }: FtrProviderContext) { } ); - // FLAKY: https://github.com/elastic/kibana/issues/120056 - registry.when.skip( + registry.when( 'Instance details when data is loaded', { config: 'basic', archives: [archiveName] }, () => { - describe.skip('fetch instance details', () => { + describe('fetch instance details', () => { let response: { status: number; body: ServiceOverviewInstanceDetails; @@ -70,7 +69,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { endpoint: 'GET /internal/apm/services/{serviceName}/service_overview_instances/details/{serviceNodeName}', params: { - path: { serviceName: 'opbeans-node', serviceNodeName: serviceNodeIds[0] }, + path: { serviceName: 'opbeans-java', serviceNodeName: serviceNodeIds[0] }, query: { start, end, diff --git a/x-pack/test/functional/apps/apm/correlations/failed_transaction_correlations.ts b/x-pack/test/functional/apps/apm/correlations/failed_transaction_correlations.ts index 592edff82eeda..84913086f3c89 100644 --- a/x-pack/test/functional/apps/apm/correlations/failed_transaction_correlations.ts +++ b/x-pack/test/functional/apps/apm/correlations/failed_transaction_correlations.ts @@ -25,8 +25,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { }; describe('failed transactions correlations', () => { - // FLAKY: https://github.com/elastic/kibana/issues/127416 - describe.skip('space with no features disabled', () => { + describe('space with no features disabled', () => { before(async () => { await esArchiver.load('x-pack/test/functional/es_archives/infra/8.0.0/metrics_and_apm'); await spacesService.create({ @@ -64,7 +63,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { const fromTime = 'Jul 29, 2019 @ 00:00:00.000'; const toTime = 'Jul 30, 2019 @ 00:00:00.000'; - await PageObjects.timePicker.setAbsoluteRange(fromTime, toTime); + await PageObjects.timePicker.setAbsoluteRange(fromTime, toTime, true); await retry.try(async () => { const apmMainContainerText = await testSubjects.getVisibleTextAll('apmMainContainer'); diff --git a/x-pack/test/functional/apps/apm/correlations/latency_correlations.ts b/x-pack/test/functional/apps/apm/correlations/latency_correlations.ts index 5cba074eedbec..88f4e2be7d7c3 100644 --- a/x-pack/test/functional/apps/apm/correlations/latency_correlations.ts +++ b/x-pack/test/functional/apps/apm/correlations/latency_correlations.ts @@ -26,10 +26,10 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { }; describe('latency correlations', () => { - // FLAKY: https://github.com/elastic/kibana/issues/127431 - describe.skip('space with no features disabled', () => { + describe('space with no features disabled', () => { before(async () => { await esArchiver.load('x-pack/test/functional/es_archives/infra/8.0.0/metrics_and_apm'); + await spacesService.delete('custom_space'); await spacesService.create({ id: 'custom_space', name: 'custom_space', @@ -65,7 +65,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { const fromTime = 'Jul 29, 2019 @ 00:00:00.000'; const toTime = 'Jul 30, 2019 @ 00:00:00.000'; - await PageObjects.timePicker.setAbsoluteRange(fromTime, toTime); + await PageObjects.timePicker.setAbsoluteRange(fromTime, toTime, true); await retry.try(async () => { const apmMainContainerText = await testSubjects.getVisibleTextAll('apmMainContainer'); @@ -121,7 +121,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { // The default tab 'Trace samples' should show the log log chart without the correlations analysis part. // First assert that the log log chart and its header are present const apmTransactionDistributionChartTitle = await testSubjects.getVisibleText( - 'apmTransactionDistributionChartTitle' + 'apmDurationDistributionChartWithScrubberTitle' ); expect(apmTransactionDistributionChartTitle).to.be(testData.logLogChartTitle); await testSubjects.existOrFail('apmCorrelationsChart'); diff --git a/x-pack/test/functional/apps/apm/feature_controls/apm_security.ts b/x-pack/test/functional/apps/apm/feature_controls/apm_security.ts index 3a0e4046291e4..b5a206a43aeb6 100644 --- a/x-pack/test/functional/apps/apm/feature_controls/apm_security.ts +++ b/x-pack/test/functional/apps/apm/feature_controls/apm_security.ts @@ -137,8 +137,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { }); }); - // FLAKY: https://github.com/elastic/kibana/issues/122001 - describe.skip('no apm privileges', () => { + describe('no apm privileges', () => { before(async () => { await security.role.create('no_apm_privileges_role', { elasticsearch: {