diff --git a/scripts/rewrite_buildkite_yamls.js b/scripts/rewrite_buildkite_yamls.js deleted file mode 100644 index d1736d3f2845a..0000000000000 --- a/scripts/rewrite_buildkite_yamls.js +++ /dev/null @@ -1,10 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -require('../src/setup_node_env'); -require('../src/dev/rewrite_buildkite_agent_rules'); diff --git a/src/dev/rewrite_buildkite_agent_rules.ts b/src/dev/rewrite_buildkite_agent_rules.ts deleted file mode 100644 index a86c4100e3a96..0000000000000 --- a/src/dev/rewrite_buildkite_agent_rules.ts +++ /dev/null @@ -1,247 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import fs from 'fs'; -import { readFile, writeFile } from 'fs/promises'; -import { resolve } from 'path'; - -import globby from 'globby'; -import yaml from 'js-yaml'; - -import { run } from '@kbn/dev-cli-runner'; -import { REPO_ROOT } from '@kbn/repo-info'; -import { ToolingLog } from '@kbn/tooling-log'; - -interface BuildkiteStepFull { - agents: { queue: string }; -} -interface BuildkiteStepPartial { - agents?: { queue?: string }; -} - -interface KBAgentDef { - queue: string; - name: string; - machineType: string; - minimumAgents?: number; - maximumAgents?: number; - idleTimeoutMins?: number; - // exitAfterOneJob?: boolean; - disableExternalIp?: boolean; - localSsds?: number; - buildPath?: string; - diskType?: string; - diskSizeGb?: number; - spot?: boolean; - zones?: string[]; - nestedVirtualization?: boolean; -} -type KibanaBuildkiteAgentLookup = Record; - -interface GobldGCPConfig { - provider: 'gcp'; - assignExternalIP?: boolean; - buildDirectory?: string; - diskSizeGb?: number; - diskType?: string; - enableSecureBoot?: boolean; - enableNestedVirtualization?: boolean; - image: string; - localSsds?: number; - localSsdInterface?: string; - machineType: string; - minCpuPlatform?: string; - imageProject: string; - networkTags?: string[]; - preemptible?: boolean; - schedulingNodeAffinity?: Record; - serviceAccount?: string; - zones?: string[]; -} - -const DRY_RUN = process.argv.includes('--dry-run'); - -if (!fs.existsSync('data/agents.json')) { - throw new Error( - 'data/agents.json does not exist - download it from https://github.com/elastic/kibana-buildkite/blob/main/agents.json' - ); -} - -/** - * Finds all .yml files in the .buildkite folder, - * rewrites all agent targeting rules from the shorthands to the full targeting syntax - */ -run( - async ({ log, flags }) => { - const paths = await globby('.buildkite/**/*.yml', { - cwd: REPO_ROOT, - onlyFiles: true, - gitignore: true, - }); - - const failedRewrites: Array<{ path: string; error: Error }> = []; - - const rewritePromises: Array> = paths.map((ymlPath) => { - return rewriteFile(ymlPath, log).catch((e) => { - // eslint-disable-next-line no-console - console.error('Failed to rewrite: ' + ymlPath, e); - failedRewrites.push({ - path: ymlPath, - error: e, - }); - }); - }); - - await Promise.all(rewritePromises); - - log.info(`Rewriting definitions complete with ${failedRewrites.length} errors.`); - - if (failedRewrites.length) { - log.warning('Failed rewrites:', ...failedRewrites); - } - - log.success('Done!'); - }, - { - flags: { - allowUnexpected: true, - }, - description: ` - Rewrites all agent targeting rules from the shorthands to the full targeting syntax - `, - } -).catch((err) => { - // eslint-disable-next-line no-console - console.error('Failure:', err); - process.exit(1); -}); - -async function rewriteFile(ymlPath: string, log: ToolingLog) { - let file = await readFile(resolve(REPO_ROOT, ymlPath), 'utf-8'); - - log.info('Loading: ' + ymlPath); - const doc = yaml.safeLoad(file); - - if (!doc.steps) { - log.info('No steps, skipping: ' + ymlPath); - return; - } - - for (const step of doc.steps as BuildkiteStepPartial[]) { - if (isQueueTargetingRule(step) && !step.agents.queue.startsWith('kb-static')) { - log.info('Rewriting: ' + ymlPath, step); - file = editYmlInPlace(file, ['agents:', `queue: ${step.agents.queue}`], () => { - return yaml.safeDump({ agents: getFullAgentTargetingRule(step.agents.queue) }).split('\n'); - }); - } - } - - if (DRY_RUN) { - await writeFile(resolve(REPO_ROOT, ymlPath + '.new'), file); - } else { - await writeFile(resolve(REPO_ROOT, ymlPath), file); - } -} - -function editYmlInPlace( - ymlContentString: string, - matchLines: Array, - editFn: (lines: string[]) => string[] -) { - const lines = ymlContentString.split('\n'); - const matchLength = matchLines.length; - - for (let i = 0; i < lines.length; i++) { - const inspectedLines = lines.slice(i, i + matchLength); - if (inspectedLines.every((l, j) => l.match(matchLines[j]) && !l.trim().startsWith('#'))) { - const indent = inspectedLines[0]?.match(/^\s+/)?.[0] || ''; - const editedLines = editFn(lines); - if (editedLines.at(-1) === '') { - editedLines.pop(); - } - - lines.splice(i, matchLength, ...editedLines.map((e) => indent + e)); - i += editedLines.length - 1; - } - } - - return lines.join('\n'); -} - -let agentNameUpdateMap: KibanaBuildkiteAgentLookup; -function getFullAgentTargetingRule(queue: string): GobldGCPConfig { - if (!agentNameUpdateMap) { - const agents = JSON.parse(fs.readFileSync('data/agents.json', 'utf8')); - agentNameUpdateMap = agents.gcp.agents.reduce( - (acc: KibanaBuildkiteAgentLookup, agent: KBAgentDef) => { - acc[agent.queue] = agent; - return acc; - }, - {} - ); - } - - const agent = agentNameUpdateMap[queue]; - if (!agent) { - throw new Error(`Unknown agent: ${queue}`); - } - - // Mapping based on expected fields in: - // https://github.com/elastic/ci/blob/main/docs/gobld/providers.mdx#L96 - return removeMissing({ - provider: 'gcp', - image: 'family/kibana-ubuntu-2004', - imageProject: 'elastic-images-qa', - diskSizeGb: agent.diskSizeGb, - diskType: agent.diskType, - enableNestedVirtualization: agent.nestedVirtualization, - localSsds: agent.localSsds, - machineType: agent.machineType, - preemptible: agent.spot, - buildDirectory: agent.buildPath, - assignExternalIP: getAssignExternalIP(agent), - }); -} - -function isQueueTargetingRule(step: BuildkiteStepPartial): step is BuildkiteStepFull & boolean { - return !!( - step.agents && - Object.keys(step.agents).length === 1 && - Object.keys(step.agents)[0] === 'queue' - ); -} - -function getAssignExternalIP(agent: KBAgentDef) { - // This is true by default for GCP, but false for Kibana-buildkite (assignExternalIp=false). - let assignExternalIp = true; - - if (agent.disableExternalIp === true) { - assignExternalIp = false; - } else if (agent.disableExternalIp === false) { - assignExternalIp = true; - } else { - // Omitted agent.disableExternalIp means disableExternalIp=true => assignExternalIp=false - assignExternalIp = false; - } - - // And when the resulting value is true, we can omit it, to rely on the default, and keep the config clean. - if (assignExternalIp) { - return undefined; - } else { - return false; - } -} - -function removeMissing(obj: T): T { - return Object.entries(obj).reduce((acc, [key, value]) => { - if (value !== null && typeof value !== 'undefined') { - acc[key] = value; - } - return acc; - }, {} as any); -}