Skip to content

Commit

Permalink
refactor failed_tests_reporter to use TS, no octokit (elastic#46993)
Browse files Browse the repository at this point in the history
* refactor failed_tests_reporter to use TS, no octokit

* update renovate config

* ensure that all kbn-test files are in ts project

* fix some type errors

* add some more tests

* [kbn-test/githubapi] cleanup and document

* collect log messages as strings instead of message objects

* ensure issue is open when updating body

* improve readability of getKibanaIssues

* expose axios helpers from dev-utils

* fix request params for fetching github issues and validate locally

* include a README for failed_tests_reporter

* improve axios error helpers

# Conflicts:
#	package.json
#	packages/kbn-test/src/index.ts
#	packages/kbn-test/src/mocha/junit_report_generation.js
#	renovate.json5
#	src/dev/jest/junit_reporter.js
#	yarn.lock
  • Loading branch information
Spencer committed Oct 8, 2019
1 parent beb6a5e commit 859910d
Show file tree
Hide file tree
Showing 40 changed files with 1,091 additions and 564 deletions.
2 changes: 1 addition & 1 deletion Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,6 @@ def buildXpack() {
def runErrorReporter() {
bash """
source src/dev/ci_setup/setup_env.sh
node src/dev/failed_tests/cli
node scripts/report_failed_tests
"""
}
1 change: 0 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,6 @@
"@kbn/utility-types": "1.0.0",
"@microsoft/api-documenter": "7.2.1",
"@microsoft/api-extractor": "7.1.8",
"@octokit/rest": "^15.10.0",
"@percy/agent": "^0.7.2",
"@types/angular": "^1.6.56",
"@types/angular-mocks": "^1.7.0",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,15 +28,9 @@ export interface AxiosResponseError<T> extends AxiosError {
}

export const isAxiosRequestError = (error: any): error is AxiosRequestError => {
return error && error.code === undefined && error.response === undefined;
return error && error.config && error.response === undefined;
};

export const isAxiosResponseError = (error: any): error is AxiosResponseError<any> => {
return error && error.code !== undefined && error.response !== undefined;
};

export const isConcliftOnGetError = (error: any) => {
return (
isAxiosResponseError(error) && error.config.method === 'GET' && error.response.status === 409
);
export const isAxiosResponseError = <T = any>(error: any): error is AxiosResponseError<T> => {
return error && error.response && error.response.status !== undefined;
};
20 changes: 20 additions & 0 deletions packages/kbn-dev-utils/src/axios/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

export * from './errors';
8 changes: 7 additions & 1 deletion packages/kbn-dev-utils/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,15 @@
*/

export { withProcRunner } from './proc_runner';
export { ToolingLog, ToolingLogTextWriter, pickLevelFromFlags } from './tooling_log';
export {
ToolingLog,
ToolingLogTextWriter,
pickLevelFromFlags,
ToolingLogCollectingWriter,
} from './tooling_log';
export { createAbsolutePathSerializer } from './serializers';
export { CA_CERT_PATH, ES_KEY_PATH, ES_CERT_PATH } from './certs';
export { run, createFailError, createFlagError, combineErrors, isFailError, Flags } from './run';
export { REPO_ROOT } from './constants';
export { KbnClient } from './kbn_client';
export * from './axios';
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,15 @@ import Url from 'url';

import Axios from 'axios';

import { isAxiosRequestError, isConcliftOnGetError } from './errors';
import { isAxiosRequestError, isAxiosResponseError } from '../axios';
import { ToolingLog } from '../tooling_log';

const isConcliftOnGetError = (error: any) => {
return (
isAxiosResponseError(error) && error.config.method === 'GET' && error.response.status === 409
);
};

export const uriencode = (
strings: TemplateStringsArray,
...values: Array<string | number | boolean>
Expand Down
1 change: 1 addition & 0 deletions packages/kbn-dev-utils/src/tooling_log/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,4 @@
export { ToolingLog } from './tooling_log';
export { ToolingLogTextWriter, ToolingLogTextWriterConfig } from './tooling_log_text_writer';
export { pickLevelFromFlags, LogLevel } from './log_levels';
export { ToolingLogCollectingWriter } from './tooling_log_collecting_writer';
Original file line number Diff line number Diff line change
Expand Up @@ -17,27 +17,20 @@
* under the License.
*/

import Octokit from '@octokit/rest';
import { markdownMetadata } from './metadata';
import { ToolingLogTextWriter } from './tooling_log_text_writer';

export { markdownMetadata };
export class ToolingLogCollectingWriter extends ToolingLogTextWriter {
messages: string[] = [];

export function getGithubClient() {
const client = new Octokit();
client.authenticate({
type: 'token',
token: process.env.GITHUB_TOKEN
});

return client;
}

export async function paginate(client, promise) {
let response = await promise;
let { data } = response;
while (client.hasNextPage(response)) {
response = await client.getNextPage(response);
data = data.concat(response.data);
constructor() {
super({
level: 'verbose',
writeTo: {
write: msg => {
// trim trailing new line
this.messages.push(msg.slice(0, -1));
},
},
});
}
return data;
}
2 changes: 1 addition & 1 deletion packages/kbn-dev-utils/tsconfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,5 @@
},
"include": [
"src/**/*"
],
]
}
8 changes: 7 additions & 1 deletion packages/kbn-test/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,17 +12,23 @@
"devDependencies": {
"@babel/cli": "^7.5.5",
"@kbn/babel-preset": "1.0.0",
"@kbn/dev-utils": "1.0.0"
"@kbn/dev-utils": "1.0.0",
"@types/parse-link-header": "^1.0.0",
"@types/strip-ansi": "^5.2.1",
"@types/xml2js": "^0.4.5"
},
"dependencies": {
"chalk": "^2.4.2",
"dedent": "^0.7.0",
"del": "^4.0.0",
"getopts": "^2.2.4",
"glob": "^7.1.2",
"parse-link-header": "^1.0.1",
"strip-ansi": "^5.2.0",
"rxjs": "^6.2.1",
"tar-fs": "^1.16.2",
"tmp": "^0.1.0",
"xml2js": "^0.4.22",
"zlib": "^1.0.5"
}
}
21 changes: 21 additions & 0 deletions packages/kbn-test/src/failed_tests_reporter/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
# failed tests reporter

A little CLI that runs in CI to find the failed tests in the JUnit reports, then create/update github issues for each failure.

## Test this script locally

To fetch some JUnit reports from a recent build on CI, visit its `Google Cloud Storage Upload Report` and execute the following in the JS Console:

```js
copy(`wget "${Array.from($$('a[href$=".xml"]')).filter(a => a.innerText === 'Download').map(a => a.href.replace('https://storage.cloud.google.com/', 'https://storage.googleapis.com/')).join('" "')}"`)
```

This copies a script to download the reporets, which can be executed in the `test/junit` directory.

Next, run the CLI in `--dry-run` mode so that it doesn't actually communicate with Github.

```sh
node scripts/report_failed_tests.js --verbose --dry-run --build-url foo
```

If you specify the `GITHUB_TOKEN` environment variable then `--dry-run` will execute read operations but still won't execute write operations.
91 changes: 91 additions & 0 deletions packages/kbn-test/src/failed_tests_reporter/get_failures.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import { ToolingLog } from '@kbn/dev-utils';

import { getFailures } from './get_failures';

const log = new ToolingLog();

it('discovers failures in ftr report', async () => {
const failures = await getFailures(log, require.resolve('./__fixtures__/ftr_report.xml'));
expect(failures).toMatchInlineSnapshot(`
Array [
Object {
"classname": "Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps/sample_data·js",
"failure": "
Error: retry.try timeout: TimeoutError: Waiting for element to be located By(css selector, [data-test-subj~=\\"layerTocActionsPanelToggleButtonRoad_Map_-_Bright\\"])
Wait timed out after 10055ms
at /var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/node_modules/selenium-webdriver/lib/webdriver.js:834:17
at process._tickCallback (internal/process/next_tick.js:68:7)
at lastError (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/test/common/services/retry/retry_for_success.ts:28:9)
at onFailure (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/test/common/services/retry/retry_for_success.ts:68:13)
",
"name": "maps app maps loaded from sample data ecommerce \\"before all\\" hook",
"time": "154.378",
},
]
`);
});

it('discovers failures in jest report', async () => {
const failures = await getFailures(log, require.resolve('./__fixtures__/jest_report.xml'));
expect(failures).toMatchInlineSnapshot(`
Array [
Object {
"classname": "X-Pack Jest Tests.x-pack/legacy/plugins/code/server/lsp",
"failure": "
TypeError: Cannot read property '0' of undefined
at Object.<anonymous>.test (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/legacy/plugins/code/server/lsp/abstract_launcher.test.ts:166:10)
",
"name": "launcher can reconnect if process died",
"time": "7.060",
},
]
`);
});

it('discovers failures in karma report', async () => {
const failures = await getFailures(log, require.resolve('./__fixtures__/karma_report.xml'));
expect(failures).toMatchInlineSnapshot(`
Array [
Object {
"classname": "Browser Unit Tests.CoordinateMapsVisualizationTest",
"failure": "Error: expected 7069 to be below 64
at Assertion.__kbnBundles__.tests../packages/kbn-expect/expect.js.Assertion.assert (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:13671:11)
at Assertion.__kbnBundles__.tests../packages/kbn-expect/expect.js.Assertion.lessThan.Assertion.below (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:13891:8)
at Function.lessThan (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:14078:15)
at _callee3$ (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:158985:60)
at tryCatch (webpack://%5Bname%5D/./node_modules/regenerator-runtime/runtime.js?:62:40)
at Generator.invoke [as _invoke] (webpack://%5Bname%5D/./node_modules/regenerator-runtime/runtime.js?:288:22)
at Generator.prototype.<computed> [as next] (webpack://%5Bname%5D/./node_modules/regenerator-runtime/runtime.js?:114:21)
at asyncGeneratorStep (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:158772:103)
at _next (http://localhost:5610/bundles/tests.bundle.js?shards=4&shard_num=1:158774:194)
",
"name": "CoordinateMapsVisualizationTest CoordinateMapsVisualization - basics should initialize OK",
"time": "0.265",
},
]
`);
});

it('discovers failures in mocha report', async () => {
const failures = await getFailures(log, require.resolve('./__fixtures__/mocha_report.xml'));
expect(failures).toMatchInlineSnapshot(`Array []`);
});
Loading

0 comments on commit 859910d

Please sign in to comment.