From 3302a828be7d2477fca78e6d657d9f0620c1aa35 Mon Sep 17 00:00:00 2001 From: Tre' Seymour Date: Mon, 14 Sep 2020 15:26:34 -0600 Subject: [PATCH] [QA][Code Coverage] Coverage teams lookup - LESS UNKNOWNS --- .ci/Jenkinsfile_coverage | 7 ++++- .../__tests__/enumerate_patterns.test.js | 12 ++++++-- .../code_coverage/ingest_coverage/index.js | 10 +++++-- .../integration_tests/ingest_coverage.test.js | 13 ++++++--- .../code_coverage/ingest_coverage/process.js | 13 ++++++--- .../team_assignment/enumerate_patterns.js | 29 ++++++++++++------- .../team_assignment/enumeration_helpers.js | 9 ++++++ .../ingest_coverage/team_assignment/index.js | 7 ++++- .../ingest_coverage/transforms.js | 14 +++++++-- ...te_team_assignments_and_ingest_coverage.sh | 6 ++-- vars/kibanaCoverage.groovy | 12 ++++---- 11 files changed, 95 insertions(+), 37 deletions(-) diff --git a/.ci/Jenkinsfile_coverage b/.ci/Jenkinsfile_coverage index 339f436e75b75..4f00f23ee5094 100644 --- a/.ci/Jenkinsfile_coverage +++ b/.ci/Jenkinsfile_coverage @@ -29,7 +29,7 @@ def handleIngestion(timestamp) { kibanaCoverage.collectVcsInfo("### Collect VCS Info") kibanaCoverage.generateReports("### Merge coverage reports") kibanaCoverage.uploadCombinedReports() - kibanaCoverage.ingest(env.JOB_NAME, BUILD_NUMBER, BUILD_URL, timestamp, previousSha, teamAssignmentsPath(), '### Ingest && Upload') + kibanaCoverage.ingest(env.JOB_NAME, BUILD_NUMBER, BUILD_URL, timestamp, previousSha, teamAssignmentsPath(), unknownTeamsLogPath(), '### Ingest && Upload') kibanaCoverage.uploadCoverageStaticSite(timestamp) } @@ -53,3 +53,8 @@ def teamAssignmentsPath() { return 'src/dev/code_coverage/ingest_coverage/team_assignment/team_assignments.txt' } +def unknownTeamsLogPath() { + return 'src/dev/code_coverage/ingest_coverage/team_assignment/unknown_team_paths.txt' +} + + diff --git a/src/dev/code_coverage/ingest_coverage/__tests__/enumerate_patterns.test.js b/src/dev/code_coverage/ingest_coverage/__tests__/enumerate_patterns.test.js index 60abab3da22b5..43db3606066f9 100644 --- a/src/dev/code_coverage/ingest_coverage/__tests__/enumerate_patterns.test.js +++ b/src/dev/code_coverage/ingest_coverage/__tests__/enumerate_patterns.test.js @@ -20,15 +20,21 @@ import expect from '@kbn/expect'; import { enumeratePatterns } from '../team_assignment/enumerate_patterns'; import { ToolingLog, REPO_ROOT } from '@kbn/dev-utils'; +import { resolve } from 'path'; +import shell from 'shelljs'; +import { tryCatch } from '../either'; const log = new ToolingLog({ level: 'info', writeTo: process.stdout, }); +const notFoundLogPath = + 'src/dev/code_coverage/ingest_coverage/team_assignment/not_found_team_assignments.txt'; +const resolved = resolve(REPO_ROOT, notFoundLogPath); describe(`enumeratePatterns`, () => { - it(`should resolve x-pack/plugins/reporting/server/browsers/extract/unzip.js to kibana-reporting`, () => { - const actual = enumeratePatterns(REPO_ROOT)(log)( + it(`should resolve to kibana-reporting`, () => { + const actual = enumeratePatterns(resolved)(log)( new Map([['x-pack/plugins/reporting', ['kibana-reporting']]]) ); @@ -37,5 +43,7 @@ describe(`enumeratePatterns`, () => { 'x-pack/plugins/reporting/server/browsers/extract/unzip.js kibana-reporting' ) ).to.be(true); + + tryCatch(() => shell.rm(resolved)); }); }); diff --git a/src/dev/code_coverage/ingest_coverage/index.js b/src/dev/code_coverage/ingest_coverage/index.js index 3bb9ce2f38bc5..3687aaa50f6a7 100644 --- a/src/dev/code_coverage/ingest_coverage/index.js +++ b/src/dev/code_coverage/ingest_coverage/index.js @@ -23,11 +23,12 @@ import { run, createFlagError } from '@kbn/dev-utils'; const ROOT = resolve(__dirname, '../../../..'); const flags = { - string: ['path', 'verbose', 'vcsInfoPath', 'teamAssignmentsPath'], + string: ['path', 'verbose', 'vcsInfoPath', 'teamAssignmentsPath', 'unknownTeamsLogPath'], help: ` --path Required, path to the file to extract coverage data --vcsInfoPath Required, path to the git info file (branch, sha, author, & commit msg) --teamAssignmentsPath Required, path to the team assignments data file +--unknownTeamsLogPath Required, path to the unknown teams log file `, }; @@ -39,14 +40,17 @@ export function runCoverageIngestionCli() { throw createFlagError('please provide a single --vcsInfoPath flag'); if (flags.teamAssignmentsPath === '') throw createFlagError('please provide a single --teamAssignments flag'); + if (flags.unknownTeamsLogPath === '') + throw createFlagError('please provide a single --unknownTeamsLogPath flag'); + if (flags.verbose) log.verbose(`Verbose logging enabled`); const resolveRoot = resolve.bind(null, ROOT); const jsonSummaryPath = resolveRoot(flags.path); const vcsInfoFilePath = resolveRoot(flags.vcsInfoPath); - const { teamAssignmentsPath } = flags; + const { teamAssignmentsPath, unknownTeamsLogPath } = flags; - prok({ jsonSummaryPath, vcsInfoFilePath, teamAssignmentsPath }, log); + prok({ jsonSummaryPath, vcsInfoFilePath, teamAssignmentsPath, unknownTeamsLogPath }, log); }, { description: ` diff --git a/src/dev/code_coverage/ingest_coverage/integration_tests/ingest_coverage.test.js b/src/dev/code_coverage/ingest_coverage/integration_tests/ingest_coverage.test.js index a4d07215efec1..d4d455466cefb 100644 --- a/src/dev/code_coverage/ingest_coverage/integration_tests/ingest_coverage.test.js +++ b/src/dev/code_coverage/ingest_coverage/integration_tests/ingest_coverage.test.js @@ -38,14 +38,18 @@ const env = { describe('Ingesting coverage', () => { const teamAssignmentsPath = 'src/dev/code_coverage/ingest_coverage/team_assignment/team_assignments.txt'; + const unknownTeamsLogPath = + 'src/dev/code_coverage/ingest_coverage/team_assignment/unknown_team_paths.txt'; - const verboseArgs = [ + const ingestCoveragParams = [ 'scripts/ingest_coverage.js', '--verbose', '--teamAssignmentsPath', teamAssignmentsPath, '--vcsInfoPath', 'src/dev/code_coverage/ingest_coverage/integration_tests/mocks/VCS_INFO.txt', + '--unknownTeamsLogPath', + unknownTeamsLogPath, '--path', ]; @@ -53,18 +57,19 @@ describe('Ingesting coverage', () => { const resolved = resolve(MOCKS_DIR, summaryPath); beforeAll(async () => { - const params = [ + const generateTeamAssignmentsParams = [ 'scripts/generate_team_assignments.js', '--src', '.github/CODEOWNERS', '--dest', teamAssignmentsPath, ]; - await execa(process.execPath, params, { cwd: ROOT_DIR, env }); + await execa(process.execPath, generateTeamAssignmentsParams, { cwd: ROOT_DIR, env }); }); afterAll(() => { shell.rm(teamAssignmentsPath); + shell.rm(unknownTeamsLogPath); }); describe(`staticSiteUrl`, () => { @@ -72,7 +77,7 @@ describe('Ingesting coverage', () => { const siteUrlRegex = /"staticSiteUrl":\s*(.+,)/; beforeAll(async () => { - const opts = [...verboseArgs, resolved]; + const opts = [...ingestCoveragParams, resolved]; const { stdout } = await execa(process.execPath, opts, { cwd: ROOT_DIR, env }); actualUrl = siteUrlRegex.exec(stdout)[1]; }); diff --git a/src/dev/code_coverage/ingest_coverage/process.js b/src/dev/code_coverage/ingest_coverage/process.js index 28a7c9ccd41b0..3d806be73e46d 100644 --- a/src/dev/code_coverage/ingest_coverage/process.js +++ b/src/dev/code_coverage/ingest_coverage/process.js @@ -51,10 +51,12 @@ const addPrePopulatedTimeStamp = addTimeStamp(process.env.TIME_STAMP || formatte const preamble = pipe(statsAndstaticSiteUrl, rootDirAndOrigPath, buildId, addPrePopulatedTimeStamp); const addTestRunnerAndStaticSiteUrl = pipe(testRunner, staticSite(staticSiteUrlBase)); -const transform = (jsonSummaryPath) => (log) => (vcsInfo) => (teamAssignmentsPath) => { +const transform = (jsonSummaryPath) => (log) => (vcsInfo) => (teamAssignmentsPath) => ( + unknownTeamsLogPath +) => { const objStream = jsonStream(jsonSummaryPath).on('done', noop); const itemizeVcsInfo = itemizeVcs(vcsInfo); - const assignTeams = teamAssignment(teamAssignmentsPath)(log); + const assignTeams = teamAssignment(teamAssignmentsPath)(unknownTeamsLogPath)(log); const jsonSummary$ = (_) => objStream.on('node', '!.*', _); @@ -86,7 +88,10 @@ const vcsInfoLines$ = (vcsInfoFilePath) => { return fromEvent(rl, 'line').pipe(takeUntil(fromEvent(rl, 'close'))); }; -export const prok = ({ jsonSummaryPath, vcsInfoFilePath, teamAssignmentsPath }, log) => { +export const prok = ( + { jsonSummaryPath, vcsInfoFilePath, teamAssignmentsPath, unknownTeamsLogPath }, + log +) => { validateRoot(COVERAGE_INGESTION_KIBANA_ROOT, log); logAll(jsonSummaryPath, log); @@ -96,7 +101,7 @@ export const prok = ({ jsonSummaryPath, vcsInfoFilePath, teamAssignmentsPath }, vcsInfoLines$(vcsInfoFilePath).subscribe( mutateVcsInfo(vcsInfo), (err) => log.error(err), - always(xformWithPath(vcsInfo)(teamAssignmentsPath)) + always(xformWithPath(vcsInfo)(teamAssignmentsPath)(unknownTeamsLogPath)) ); }; diff --git a/src/dev/code_coverage/ingest_coverage/team_assignment/enumerate_patterns.js b/src/dev/code_coverage/ingest_coverage/team_assignment/enumerate_patterns.js index 8615e4a7cf014..ec1dadea57e7d 100644 --- a/src/dev/code_coverage/ingest_coverage/team_assignment/enumerate_patterns.js +++ b/src/dev/code_coverage/ingest_coverage/team_assignment/enumerate_patterns.js @@ -17,8 +17,10 @@ * under the License. */ -import { readdirSync, statSync } from 'fs'; +import { readdirSync, statSync, writeFileSync } from 'fs'; import { join } from 'path'; +import { REPO_ROOT } from '@kbn/dev-utils'; + import { push, prokGlob, @@ -28,14 +30,18 @@ import { isDir, tryPath, dropEmpty, - notFound, + encoding, + collectAndLogNotFound, } from './enumeration_helpers'; import { stripLeading } from '../transforms'; -export const enumeratePatterns = (rootPath) => (log) => (patterns) => { +export const enumeratePatterns = (notFoundLogPath) => (log) => (patterns) => { + const writeToFile = writeFileSync.bind(null, notFoundLogPath); + const blank = ''; + writeToFile(blank, { encoding }); + const res = []; const resPush = push(res); - const logNotFound = notFound(log); for (const entry of patterns) { const [pathPattern, teams] = entry; @@ -43,25 +49,26 @@ export const enumeratePatterns = (rootPath) => (log) => (patterns) => { const owner = teams[0]; const existsWithOwner = pathExists(owner); - const collect = (x) => existsWithOwner(x).forEach(resPush); - tryPath(cleaned).fold(logNotFound, collect); + const collectNotFound = collectAndLogNotFound(writeToFile); + const collectFound = (x) => existsWithOwner(x).forEach(resPush); + tryPath(cleaned).fold(collectNotFound(log), collectFound); } return res; function pathExists(owner) { - const creeper = (x) => creepFsSync(x, [], rootPath, owner); + const creeper = (x) => creepFsSync(x, [], owner); return function creepAllAsGlobs(pathPattern) { return prokGlob(pathPattern).map(creeper).filter(dropEmpty); }; } }; -function creepFsSync(aPath, xs, rootPath, owner) { +function creepFsSync(aPath, xs, owner) { xs = xs || []; - const joinRoot = join.bind(null, rootPath); - const trimRoot = trim(rootPath); + const joinRoot = join.bind(null, REPO_ROOT); + const trimRoot = trim(REPO_ROOT); const joined = joinRoot(aPath); const isADir = isDir(joined); @@ -73,7 +80,7 @@ function creepFsSync(aPath, xs, rootPath, owner) { const full = isADir ? join(aPath, entry) : entry; const fullIsDir = statSync(full).isDirectory(); - if (fullIsDir && !isBlackListedDir(full)) xs = creepFsSync(full, xs, rootPath, owner); + if (fullIsDir && !isBlackListedDir(full)) xs = creepFsSync(full, xs, owner); else if (isWhiteListedFile(full)) xs.push(`${trimRoot(full)} ${owner}`); } } diff --git a/src/dev/code_coverage/ingest_coverage/team_assignment/enumeration_helpers.js b/src/dev/code_coverage/ingest_coverage/team_assignment/enumeration_helpers.js index 44f50ce95e787..7d5ab7b4447dc 100644 --- a/src/dev/code_coverage/ingest_coverage/team_assignment/enumeration_helpers.js +++ b/src/dev/code_coverage/ingest_coverage/team_assignment/enumeration_helpers.js @@ -21,6 +21,7 @@ import { statSync } from 'fs'; import isGlob from 'is-glob'; import glob from 'glob'; import { left, right, tryCatch } from '../either'; +import { pipe } from '../utils'; export const push = (xs) => (x) => xs.push(x); export const pathExists = (x) => tryCatch(() => statSync(x)).fold(left, right); @@ -44,3 +45,11 @@ export const tryPath = (x) => { }; export const dropEmpty = (x) => x.length > 0; export const notFound = (log) => (err) => log.error(`\n!!! Not Found: \n${err}`); +export const encoding = 'utf8'; +export const appendUtf8 = { flag: 'a', encoding }; +const flushToLogFile = (fileWriteFn) => (x) => { + fileWriteFn(`${x}\n`, appendUtf8); + return x; +}; +export const collectAndLogNotFound = (fileWrite) => (log) => + pipe(flushToLogFile(fileWrite), notFound(log)); diff --git a/src/dev/code_coverage/ingest_coverage/team_assignment/index.js b/src/dev/code_coverage/ingest_coverage/team_assignment/index.js index af3b2e8ccbd00..f544520b6ce62 100644 --- a/src/dev/code_coverage/ingest_coverage/team_assignment/index.js +++ b/src/dev/code_coverage/ingest_coverage/team_assignment/index.js @@ -23,6 +23,7 @@ import { flush } from './flush'; import { enumeratePatterns } from './enumerate_patterns'; import { push } from './enumeration_helpers'; import { pipe } from '../utils'; +import { resolve } from 'path'; const flags = { string: ['src', 'dest'], @@ -47,7 +48,7 @@ export const generateTeamAssignments = () => { () => pipe( logSuccess(flags.src, log), - enumeratePatterns(REPO_ROOT)(log), + enumeratePatterns(notFoundPath())(log), flush(flags.dest)(log) )(new Map(data)) ); @@ -71,3 +72,7 @@ function logSuccess(src, log) { return dataObj; }; } +function notFoundPath() { + const x = 'src/dev/code_coverage/ingest_coverage/team_assignment/not_found_team_assignments.txt'; + return resolve(REPO_ROOT, x); +} diff --git a/src/dev/code_coverage/ingest_coverage/transforms.js b/src/dev/code_coverage/ingest_coverage/transforms.js index d88ab08222f49..fda76967788b0 100644 --- a/src/dev/code_coverage/ingest_coverage/transforms.js +++ b/src/dev/code_coverage/ingest_coverage/transforms.js @@ -22,6 +22,8 @@ import * as Maybe from './maybe'; import { always, id, noop, pink } from './utils'; import execa from 'execa'; import { resolve } from 'path'; +import { appendUtf8 } from './team_assignment/enumeration_helpers'; +import { writeFileSync } from 'fs'; const ROOT_DIR = resolve(__dirname, '../../../..'); @@ -107,22 +109,28 @@ const findTeam = (x) => x.match(/.+\s{1,3}(.+)$/, 'gm'); export const pluckIndex = (idx) => (xs) => xs[idx]; const pluckTeam = pluckIndex(1); -export const teamAssignment = (teamAssignmentsPath) => (log) => async (obj) => { +export const teamAssignment = (teamAssignmentsPath) => (unknownTeamsLogPath) => (log) => async ( + obj +) => { const { coveredFilePath } = obj; const isTotal = Either.fromNullable(obj.isTotal); - return isTotal.isRight() ? obj : await assignTeam(teamAssignmentsPath, coveredFilePath, log, obj); + return isTotal.isRight() + ? obj + : await assignTeam(teamAssignmentsPath, coveredFilePath, unknownTeamsLogPath, log, obj); }; -async function assignTeam(teamAssignmentsPath, coveredFilePath, log, obj) { +async function assignTeam(teamAssignmentsPath, coveredFilePath, unknownTeamsLogPath, log, obj) { const params = [coveredFilePath, teamAssignmentsPath]; let grepResponse; + const logUnknowns = writeFileSync.bind(null, unknownTeamsLogPath); try { const { stdout } = await execa('grep', params, { cwd: ROOT_DIR }); grepResponse = stdout; } catch (e) { log.error(`\n!!! Unknown Team for path: \n\t\t${pink(coveredFilePath)}\n`); + logUnknowns(`${coveredFilePath}\n`, appendUtf8); } return Either.fromNullable(grepResponse) diff --git a/src/dev/code_coverage/shell_scripts/generate_team_assignments_and_ingest_coverage.sh b/src/dev/code_coverage/shell_scripts/generate_team_assignments_and_ingest_coverage.sh index 62b81929ae79b..b6e4372c7fb0a 100644 --- a/src/dev/code_coverage/shell_scripts/generate_team_assignments_and_ingest_coverage.sh +++ b/src/dev/code_coverage/shell_scripts/generate_team_assignments_and_ingest_coverage.sh @@ -29,6 +29,8 @@ export DELAY TEAM_ASSIGN_PATH=$5 +UNKNOWN_TEAMS_LOG_PATH=$6 + # Build team assignments dat file node scripts/generate_team_assignments.js --verbose --src .github/CODEOWNERS --dest $TEAM_ASSIGN_PATH @@ -37,14 +39,14 @@ for x in jest functional; do COVERAGE_SUMMARY_FILE=target/kibana-coverage/${x}-combined/coverage-summary.json - node scripts/ingest_coverage.js --verbose --path ${COVERAGE_SUMMARY_FILE} --vcsInfoPath ./VCS_INFO.txt --teamAssignmentsPath $TEAM_ASSIGN_PATH + node scripts/ingest_coverage.js --verbose --path ${COVERAGE_SUMMARY_FILE} --vcsInfoPath ./VCS_INFO.txt --teamAssignmentsPath $TEAM_ASSIGN_PATH --unknownTeamsLogPath $UNKNOWN_TEAMS_LOG_PATH done # Need to override COVERAGE_INGESTION_KIBANA_ROOT since mocha json file has original intake worker path COVERAGE_SUMMARY_FILE=target/kibana-coverage/mocha-combined/coverage-summary.json export COVERAGE_INGESTION_KIBANA_ROOT=/dev/shm/workspace/kibana -node scripts/ingest_coverage.js --verbose --path ${COVERAGE_SUMMARY_FILE} --vcsInfoPath ./VCS_INFO.txt --teamAssignmentsPath $TEAM_ASSIGN_PATH +node scripts/ingest_coverage.js --verbose --path ${COVERAGE_SUMMARY_FILE} --vcsInfoPath ./VCS_INFO.txt --teamAssignmentsPath $TEAM_ASSIGN_PATH --unknownTeamsLogPath $UNKNOWN_TEAMS_LOG_PATH echo "### Ingesting Code Coverage - Complete" echo "" diff --git a/vars/kibanaCoverage.groovy b/vars/kibanaCoverage.groovy index e75ed8fef9875..210ac2e12e6bb 100644 --- a/vars/kibanaCoverage.groovy +++ b/vars/kibanaCoverage.groovy @@ -169,31 +169,31 @@ def uploadCombinedReports() { ) } -def ingestData(jobName, buildNum, buildUrl, previousSha, teamAssignmentsPath, title) { +def ingestData(jobName, buildNum, buildUrl, previousSha, teamAssignmentsPath, unknownTeamsLogPath, title) { kibanaPipeline.bash(""" source src/dev/ci_setup/setup_env.sh yarn kbn bootstrap --prefer-offline # Using existing target/kibana-coverage folder - . src/dev/code_coverage/shell_scripts/generate_team_assignments_and_ingest_coverage.sh '${jobName}' ${buildNum} '${buildUrl}' '${previousSha}' '${teamAssignmentsPath}' + . src/dev/code_coverage/shell_scripts/generate_team_assignments_and_ingest_coverage.sh '${jobName}' ${buildNum} '${buildUrl}' '${previousSha}' '${teamAssignmentsPath}' '${unknownTeamsLogPath}' """, title) } -def ingestWithVault(jobName, buildNum, buildUrl, previousSha, teamAssignmentsPath, title) { +def ingestWithVault(jobName, buildNum, buildUrl, previousSha, teamAssignmentsPath, unknownTeamsLogPath, title) { def vaultSecret = 'secret/kibana-issues/prod/coverage/elasticsearch' withVaultSecret(secret: vaultSecret, secret_field: 'host', variable_name: 'HOST_FROM_VAULT') { withVaultSecret(secret: vaultSecret, secret_field: 'username', variable_name: 'USER_FROM_VAULT') { withVaultSecret(secret: vaultSecret, secret_field: 'password', variable_name: 'PASS_FROM_VAULT') { - ingestData(jobName, buildNum, buildUrl, previousSha, teamAssignmentsPath, title) + ingestData(jobName, buildNum, buildUrl, previousSha, teamAssignmentsPath, unknownTeamsLogPath, title) } } } } -def ingest(jobName, buildNumber, buildUrl, timestamp, previousSha, teamAssignmentsPath, title) { +def ingest(jobName, buildNumber, buildUrl, timestamp, previousSha, teamAssignmentsPath, unknownTeamsLogPath, title) { withEnv([ "TIME_STAMP=${timestamp}", ]) { - ingestWithVault(jobName, buildNumber, buildUrl, previousSha, teamAssignmentsPath, title) + ingestWithVault(jobName, buildNumber, buildUrl, previousSha, teamAssignmentsPath, unknownTeamsLogPath, title) } }