From 279d849ce67fa36e5861785032a59ee3e25f2c4b Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Wed, 23 Feb 2022 19:05:17 +0100 Subject: [PATCH 01/20] chore(tooling): use typescript for all scripts --- clients.config.json | 8 ++ package.json | 6 +- scripts/buildClients.ts | 40 ++++++ scripts/buildSpecs.ts | 70 ++++++++++ scripts/builds/clients.sh | 45 ------- scripts/builds/specs.sh | 58 -------- scripts/common.ts | 87 ++++++++++++ scripts/formatter.sh | 59 -------- scripts/formatter.ts | 33 +++++ scripts/generate.sh | 92 ------------- scripts/generate.ts | 68 ++++++++++ scripts/index.ts | 187 ++++++++++++++++++++++++++ scripts/oraLog.ts | 60 +++++++++ scripts/package.json | 9 +- scripts/post-gen/global.sh | 4 + scripts/post-gen/java.sh | 4 + scripts/post-gen/javascript.sh | 5 + scripts/post-gen/php.sh | 5 + scripts/pre-gen/java.sh | 4 + scripts/pre-gen/setHostsOptions.ts | 11 +- scripts/release/common.ts | 37 ----- scripts/types.ts | 7 + yarn.lock | 208 ++++++++++++++++++++++++++++- 23 files changed, 802 insertions(+), 305 deletions(-) create mode 100644 clients.config.json create mode 100644 scripts/buildClients.ts create mode 100644 scripts/buildSpecs.ts delete mode 100755 scripts/builds/clients.sh delete mode 100755 scripts/builds/specs.sh create mode 100644 scripts/common.ts delete mode 100755 scripts/formatter.sh create mode 100644 scripts/formatter.ts delete mode 100755 scripts/generate.sh create mode 100644 scripts/generate.ts create mode 100644 scripts/index.ts create mode 100644 scripts/oraLog.ts create mode 100644 scripts/types.ts diff --git a/clients.config.json b/clients.config.json new file mode 100644 index 0000000000..9cf7d23988 --- /dev/null +++ b/clients.config.json @@ -0,0 +1,8 @@ +{ + "java": { + "folder": "clients/algoliasearch-client-java-2" + }, + "javascript": { + "folder": "clients/algoliasearch-client-javascript" + } +} diff --git a/package.json b/package.json index 4e3413016a..994aebfcaa 100644 --- a/package.json +++ b/package.json @@ -9,9 +9,7 @@ "scripts/" ], "scripts": { - "build:clients": "./scripts/multiplexer.sh ${2:-nonverbose} ./scripts/builds/clients.sh ${0:-all} ${1:-all}", - "build:specs": "./scripts/builds/specs.sh ${0:-all} ${1:-yml}", - "build": "yarn build:specs && yarn build:clients", + "api": "yarn workspace scripts ts-node ./index.ts", "clean": "rm -rf **/dist **/build **/node_modules **/.gradle", "cts:generate": "yarn workspace tests build && ./scripts/multiplexer.sh ${2:-nonverbose} yarn workspace tests generate ${0:-all} ${1:-all}", "cts:test": "./scripts/multiplexer.sh ${1:-nonverbose} ./scripts/runCTS.sh ${0:-all} all", @@ -23,8 +21,6 @@ "docker:setup": "yarn docker:clean && yarn docker:build && yarn docker:mount", "docker": "docker exec -it dev yarn $*", "lint": "eslint --ext=ts .", - "post:generate": "./scripts/post-gen/global.sh", - "generate": "./scripts/multiplexer.sh ${2:-nonverbose} ./scripts/generate.sh ${0:-all} ${1:-all} && yarn post:generate ${0:-all}", "playground:browser": "yarn workspace javascript-browser-playground start", "playground": "yarn && ./scripts/multiplexer.sh ${2:-nonverbose} ./scripts/playground.sh ${0:-javascript} ${1:-search}", "specs:fix": "eslint --ext=yml specs/ --fix", diff --git a/scripts/buildClients.ts b/scripts/buildClients.ts new file mode 100644 index 0000000000..4faf0bc5ff --- /dev/null +++ b/scripts/buildClients.ts @@ -0,0 +1,40 @@ +import { run } from './common'; +import { createSpinner } from './oraLog'; +import type { Generator } from './types'; + +async function buildClient( + { language, additionalProperties: { packageName } }: Generator, + verbose: boolean +): Promise { + const spinner = createSpinner(`building ${language}`, verbose).start(); + switch (language) { + case 'javascript': + await run(`yarn workspace ${packageName} clean`, { verbose }); + await run( + `yarn workspace algoliasearch-client-javascript build ${packageName}`, + { verbose } + ); + break; + case 'java': + await run( + `./gradle/gradlew --no-daemon -p clients/${packageName} assemble`, + { + verbose, + } + ); + break; + case 'php': + break; + default: + } + spinner.succeed(); +} + +export async function buildClients( + generators: Generator[], + verbose: boolean +): Promise { + for (const gen of generators) { + await buildClient(gen, verbose); + } +} diff --git a/scripts/buildSpecs.ts b/scripts/buildSpecs.ts new file mode 100644 index 0000000000..a78787db90 --- /dev/null +++ b/scripts/buildSpecs.ts @@ -0,0 +1,70 @@ +import fsp from 'fs/promises'; + +import { hashElement } from 'folder-hash'; + +import { fileExists, run } from './common'; +import { createSpinner } from './oraLog'; + +async function buildSpec( + client: string, + outputFormat: string, + verbose: boolean, + useCache: boolean +): Promise { + const cacheFile = `../specs/dist/${client}.cache`; + if (useCache) { + // check if file and cache exist + const spinner = createSpinner( + `checking cache for ${client}`, + verbose + ).start(); + if (await fileExists(`../specs/bundled/${client}.yml`)) { + const hash = (await hashElement(`../specs/${client}`)).hash; + // compare with stored cache + if (await fileExists(cacheFile)) { + const storedHash = (await fsp.readFile(cacheFile)).toString(); + if (storedHash === hash) { + spinner.succeed( + 'skipped building spec because the files did not change' + ); + return; + } + } + } + + spinner.info(`cache not found for ${client} spec`); + } + + const spinner = createSpinner(`linting ${client} spec`, verbose).start(); + await run(`yarn specs:lint ${client}`, { verbose }); + + spinner.text = `building ${client} spec`; + await run( + `yarn openapi bundle specs/${client}/spec.yml -o specs/bundled/${client}.${outputFormat} --ext ${outputFormat}`, + { verbose } + ); + + spinner.text = `validate ${client} spec`; + await run(`yarn openapi lint specs/bundled/${client}.${outputFormat}`, { + verbose, + }); + + spinner.text = `storing ${client} cache`; + const hash = (await hashElement(`../specs/${client}`)).hash; + await fsp.writeFile(cacheFile, hash); + + spinner.succeed(); +} + +export async function buildSpecs( + clients: string[], + outputFormat: 'json' | 'yml', + verbose: boolean, + useCache: boolean +): Promise { + await fsp.mkdir('../specs/dist', { recursive: true }); + + for (const client of clients) { + await buildSpec(client, outputFormat, verbose, useCache); + } +} diff --git a/scripts/builds/clients.sh b/scripts/builds/clients.sh deleted file mode 100755 index 9d6e82491a..0000000000 --- a/scripts/builds/clients.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash - -# Break on non-zero code -set -e - -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" -# Move to the root (easier to locate other scripts) -cd ${DIR}/../.. - -LANGUAGE=$1 -CLIENT=$2 -GENERATOR="$LANGUAGE-$CLIENT" -PACKAGE=$(cat openapitools.json | jq -r --arg generator "$GENERATOR" '."generator-cli".generators[$generator].additionalProperties.packageName') - -if [[ -z $PACKAGE ]]; then - echo "Unknown package ${PACKAGE}" - exit 1 -fi - -# Commands are based on the LANGUAGE -if [[ $LANGUAGE == 'javascript' ]]; then - echo "> Cleaning previous build $GENERATOR..." - yarn workspace $PACKAGE clean - - echo "> Bundling $GENERATOR..." - CMD="yarn workspace algoliasearch-client-javascript build $PACKAGE" -elif [[ $LANGUAGE == 'php' ]]; then - # no build needed (for now) - : -elif [[ $LANGUAGE == 'java' ]]; then - CMD="./gradle/gradlew --no-daemon -p clients/$PACKAGE assemble" -fi - -if [[ $VERBOSE == "true" ]]; then - $CMD -else - set +e - log=$($CMD) - - if [[ $? != 0 ]]; then - echo "$log" - exit 1 - fi - set -e -fi diff --git a/scripts/builds/specs.sh b/scripts/builds/specs.sh deleted file mode 100755 index 3dd5bcfe94..0000000000 --- a/scripts/builds/specs.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -if [[ ! $CI ]] && [[ ! $DOCKER ]]; then - echo "You should run scripts via the docker container, see README.md" - - exit 1 -fi - -# Break on non-zero code -set -e - -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" -# Move to the root (easier to locate other scripts) -cd ${DIR}/../.. - -CLIENT=$1 -OUTPUT=$2 - -check_format_spec() { - local client=$1 - echo "> Checking format of $client spec" - yarn specs:lint $client - echo "" -} - -build_spec() { - local client=$1 - yarn openapi bundle specs/${client}/spec.yml -o specs/bundled/${client}.${OUTPUT} --ext ${OUTPUT} - echo "" -} - -validate_output_spec() { - local client=$1 - yarn openapi lint specs/bundled/${client}.${OUTPUT} - echo "" -} - -CLIENTS=($(find specs/*/spec.yml | awk -F / '{ print $(NF-1) }')) - -if [[ $CLIENT == "all" ]]; then - CLIENTS=("${CLIENTS[@]}") -elif [[ ${CLIENTS[*]} =~ ${CLIENT} ]]; then - CLIENTS=($CLIENT) -else - echo "Unknown spec ${CLIENT}" - exit 1 -fi - -if [[ $OUTPUT != "yml" ]] && [[ $OUTPUT != "json" ]]; then - echo "Unknown output ${OUTPUT}" - exit 1 -fi - -for client in "${CLIENTS[@]}"; do - check_format_spec $client - build_spec $client - validate_output_spec $client -done diff --git a/scripts/common.ts b/scripts/common.ts new file mode 100644 index 0000000000..fd52b372a2 --- /dev/null +++ b/scripts/common.ts @@ -0,0 +1,87 @@ +import fsp from 'fs/promises'; + +import execa from 'execa'; // https://github.com/sindresorhus/execa/tree/v5.1.1 + +import clientsConfig from '../clients.config.json'; +import openapitools from '../openapitools.json'; + +import type { Generator, RunOptions } from './types'; + +export const CI = Boolean(process.env.CI); +export const DOCKER = Boolean(process.env.DOCKER); + +export const GENERATORS = Object.fromEntries( + Object.entries(openapitools['generator-cli'].generators).map(([key, gen]) => { + return [key, { ...gen, ...splitGeneratorKey(key) }]; + }) +); + +export const LANGUAGES = [ + ...new Set(Object.values(GENERATORS).map((gen) => gen.language)), +]; + +export const CLIENTS = [ + ...new Set(Object.values(GENERATORS).map((gen) => gen.client)), +]; + +export const CLIENTS_JS = CLIENTS.concat([]); + +export function splitGeneratorKey(key: string): Generator { + const language = key.slice(0, key.indexOf('-')); + const client = key.slice(key.indexOf('-') + 1); + return { language, client, key }; +} + +export function createGeneratorKey({ + language, + client, +}: Pick): string { + return `${language}-${client}`; +} + +export function getLanguageFolder(language: string): string { + return clientsConfig[language].folder; +} + +export async function run( + command: string, + { errorMessage, verbose }: RunOptions = {} +): Promise { + try { + if (verbose) { + return ( + await execa.command(command, { + stdout: 'inherit', + shell: 'bash', + cwd: '../', + }) + ).stdout; + } + return (await execa.command(command, { shell: 'bash', cwd: '../' })).stdout; + } catch (err) { + if (errorMessage) { + throw new Error(`[ERROR] ${errorMessage}`); + } else { + throw err; + } + } +} + +export async function fileExists(path: string): Promise { + try { + return (await fsp.stat(path)).isFile(); + } catch { + return false; + } +} + +export async function runIfExists( + script: string, + args: string, + opts: RunOptions = {} +): Promise { + if (await fileExists(`../${script}`)) { + return await run(`${script} ${args}`, opts); + } + return ''; +} diff --git a/scripts/formatter.sh b/scripts/formatter.sh deleted file mode 100755 index 0aa4024e3c..0000000000 --- a/scripts/formatter.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - -# Break on non-zero code -set -e - -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" -# Move to the root (easier to locate other scripts) -cd ${DIR}/.. - -LANGUAGE=$1 -FOLDER=$2 - -if [[ ! -d "$FOLDER" ]]; then - echo "Output folder does not exist for $LANGUAGE, skipping..." - exit 0 -fi - -if [[ $LANGUAGE == 'javascript' ]]; then - # jsdoc/require-hyphen-before-param-description fails to lint more than - # 6 parameters, we re-run the script if failed to lint the rest - CMD="yarn eslint --ext=ts,js ${FOLDER} --fix || yarn eslint --ext=ts,js ${FOLDER} --fix" -elif [[ $LANGUAGE == 'php' ]]; then - FIXER="clients/algoliasearch-client-php/vendor/bin/php-cs-fixer" - if [[ $CI ]]; then - PHP="php" - else - PHP="php8" - fi - CMD="composer update --working-dir=clients/algoliasearch-client-php \ - && composer dump-autoload --working-dir=clients/algoliasearch-client-php \ - && PHP_CS_FIXER_IGNORE_ENV=1 $PHP $FIXER fix $FOLDER --using-cache=no --allow-risky=yes" -elif [[ $LANGUAGE == 'java' ]]; then - CMD="find $FOLDER -type f -name \"*.java\" | xargs java --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \ - --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \ - --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \ - --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \ - --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED \ - -jar /tmp/java-formatter.jar -r \ - && yarn prettier --write $FOLDER" -else - echo "Cannot format unknow language $LANGUAGE" - exit 1 -fi - -echo "> Formatting ${LANGUAGE} in ${FOLDER}..." - -if [[ $VERBOSE == "true" ]]; then - # CAREFUL WITH EVAL (not safe) - eval $CMD -else - set +e - log=$(eval $CMD) - - if [[ $? != 0 ]]; then - echo "$log" - exit 1 - fi - set -e -fi diff --git a/scripts/formatter.ts b/scripts/formatter.ts new file mode 100644 index 0000000000..ed16a7e338 --- /dev/null +++ b/scripts/formatter.ts @@ -0,0 +1,33 @@ +import { CI, run } from './common'; + +export async function formatter( + language: string, + folder: string, + verbose?: boolean +): Promise { + let cmd = ''; + switch (language) { + case 'javascript': + cmd = `yarn eslint --ext=ts,js ${folder} --fix || yarn eslint --ext=ts,js ${folder} --fix`; + break; + case 'java': + cmd = `find ${folder} -type f -name "*.java" | xargs java --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \ + --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \ + --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \ + --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \ + --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED \ + -jar /tmp/java-formatter.jar -r \ + && yarn prettier --write ${folder}`; + break; + case 'php': + cmd = `composer update --working-dir=clients/algoliasearch-client-php \ + && composer dump-autoload --working-dir=clients/algoliasearch-client-php \ + && PHP_CS_FIXER_IGNORE_ENV=1 ${ + CI ? 'php' : 'php8' + } clients/algoliasearch-client-php/vendor/bin/php-cs-fixer fix ${folder} --using-cache=no --allow-risky=yes`; + break; + default: + return; + } + await run(cmd, { verbose }); +} diff --git a/scripts/generate.sh b/scripts/generate.sh deleted file mode 100755 index bf25a6314a..0000000000 --- a/scripts/generate.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/bin/bash - -# Break on non-zero code -set -e - -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" -# Move to the root (easier to locate other scripts) -cd ${DIR}/.. - -LANGUAGE=$1 -CLIENT=$2 -GENERATOR="$1-$2" - -compute_hash() { - cacheSpec=$(find specs/$CLIENT -type f -print0 | xargs -0 sha1sum | sha1sum | tr -d ' ') - cacheCommon=$(find specs/common -type f -print0 | xargs -0 sha1sum | sha1sum | tr -d ' ') - echo "$cacheSpec$cacheCommon" -} - -# build spec before generating client -build_spec() { - # check if file and cache exist - mkdir -p specs/dist - cacheFile="specs/dist/$CLIENT.cache" - if [[ -f specs/bundled/$CLIENT.yml ]]; then - cache=$(compute_hash) - # compare with stored cache - if [[ -f $cacheFile && $(cat $cacheFile) == $cache ]]; then - echo "> Skipped building spec because the files did not change..." - return - fi - fi - yarn build:specs $CLIENT - - # store hash - cache=$(compute_hash) - echo $cache > $cacheFile -} - -# Run the pre generation script if it exists. -run_pre_gen() { - pregen="./scripts/pre-gen/${LANGUAGE}.sh" - - if [[ -f "$pregen" ]]; then - echo "> Running pre-gen script for $GENERATOR..." - $pregen $CLIENT - fi - - # Sets the hosts option to the `openapitools.json` config file - yarn workspace scripts setHostsOptions $LANGUAGE $CLIENT -} - -generate_client() { - echo "> Generating code for $GENERATOR..." - - CMD="yarn openapi-generator-cli generate --generator-key $GENERATOR" - - if [[ $VERBOSE == "true" ]]; then - $CMD - else - set +e - log=$($CMD) - - if [[ $? != 0 ]]; then - echo "$log" - exit 1 - fi - set -e - fi -} - -# Run the post generation script if it exists. -run_post_gen() { - postgen="./scripts/post-gen/${LANGUAGE}.sh" - - folder=$(cat openapitools.json | jq -r --arg generator "$GENERATOR" '."generator-cli".generators[$generator].output' | sed 's/#{cwd}\///g') - - if [[ -f "$postgen" ]]; then - echo "> Running post-gen script for $GENERATOR..." - $postgen $folder $GENERATOR - fi - - ./scripts/formatter.sh $LANGUAGE $folder -} - -if [[ ! $CI ]]; then - build_spec -fi - -run_pre_gen -generate_client -run_post_gen diff --git a/scripts/generate.ts b/scripts/generate.ts new file mode 100644 index 0000000000..965a669d3d --- /dev/null +++ b/scripts/generate.ts @@ -0,0 +1,68 @@ +import { buildSpecs } from './buildSpecs'; +import { CI, getLanguageFolder, run, runIfExists } from './common'; +import { formatter } from './formatter'; +import { createSpinner } from './oraLog'; +import { setHostsOptions } from './pre-gen/setHostsOptions'; +import type { Generator } from './types'; + +async function preGen( + { language, client, key, output }: Generator, + verbose?: boolean +): Promise { + const folder = output.replace('#{cwd}/', ''); + await runIfExists(`./scripts/pre-gen/${language}.sh`, `${folder} ${key}`, { + verbose, + }); + + await setHostsOptions({ client, key }); +} + +async function generateClient( + { key }: Generator, + verbose?: boolean +): Promise { + await run(`yarn openapi-generator-cli generate --generator-key ${key}`, { + verbose, + }); +} + +async function postGen( + { language, key, output }: Generator, + verbose?: boolean +): Promise { + const folder = output.replace('#{cwd}/', ''); + await runIfExists(`./scripts/post-gen/${language}.sh`, `${folder} ${key}`, { + verbose, + }); +} + +export async function generate( + generators: Generator[], + verbose: boolean +): Promise { + if (!CI) { + const clients = [...new Set(generators.map((gen) => gen.client))]; + await buildSpecs(clients, 'yml', verbose, true); + } + + for (const gen of generators) { + const spinner = createSpinner(`pre-gen ${gen.key}`, verbose).start(); + await preGen(gen, verbose); + + spinner.text = `generation ${gen.key}`; + await generateClient(gen, verbose); + + spinner.text = `post-gen ${gen.key}`; + await postGen(gen, verbose); + + spinner.succeed(); + } + + const langs = [...new Set(generators.map((gen) => gen.language))]; + for (const lang of langs) { + const spinner = createSpinner(`formatting ${lang}`, verbose).start(); + await formatter(lang, getLanguageFolder(lang), verbose); + + spinner.succeed(); + } +} diff --git a/scripts/index.ts b/scripts/index.ts new file mode 100644 index 0000000000..2a8d2b778e --- /dev/null +++ b/scripts/index.ts @@ -0,0 +1,187 @@ +import { Argument, program } from 'commander'; +import inquirer from 'inquirer'; + +import { buildClients } from './buildClients'; +import { buildSpecs } from './buildSpecs'; +import { + CI, + CLIENTS, + CLIENTS_JS, + createGeneratorKey, + DOCKER, + GENERATORS, + LANGUAGES, +} from './common'; +import { generate } from './generate'; +import type { Generator } from './types'; + +if (!CI && !DOCKER) { + // eslint-disable-next-line no-console + console.log('You should run scripts via the docker container, see README.md'); + // eslint-disable-next-line no-process-exit + process.exit(1); +} + +program.name('api'); + +async function promptLanguage(defaut: string | undefined): Promise { + if (defaut) { + return defaut; + } + const { language } = await inquirer.prompt([ + { + type: 'list', + name: 'language', + message: 'Select a language', + default: 'all', + choices: ['all', new inquirer.Separator()].concat(LANGUAGES), + }, + ]); + return language; +} + +async function promptClient( + defaut: string | undefined, + clientList = CLIENTS +): Promise { + if (defaut) { + return defaut; + } + const { client } = await inquirer.prompt([ + { + type: 'list', + name: 'client', + message: 'Select a client', + default: 'all', + choices: ['all', new inquirer.Separator()].concat(clientList), + }, + ]); + return client; +} + +function generatorList({ + language, + client, + clientList = CLIENTS, +}: Pick & { + clientList?: string[]; +}): Generator[] { + let langsTodo = [language]; + let clientsTodo = [client]; + if (language === 'all') { + langsTodo = LANGUAGES; + } + if (client === 'all') { + clientsTodo = clientList; + } + + return langsTodo + .flatMap((lang) => + clientsTodo.map( + (cli) => GENERATORS[createGeneratorKey({ language: lang, client: cli })] + ) + ) + .filter(Boolean); +} + +program + .command('generate') + .description('Generate a specified client') + .addArgument( + new Argument('[language]', 'The language').choices( + ['all'].concat(LANGUAGES) + ) + ) + .addArgument( + new Argument('[client]', 'The client').choices(['all'].concat(CLIENTS)) + ) + .option('-v, --verbose', 'make the generation verbose') + .action( + async ( + language: string | undefined, + client: string | undefined, + { verbose } + ) => { + // eslint-disable-next-line no-param-reassign + language = await promptLanguage(language); + // eslint-disable-next-line no-param-reassign + client = await promptClient(client); + + await generate(generatorList({ language, client }), Boolean(verbose)); + } + ); + +program + .command('build:clients') + .description('Build a specified client') + .addArgument( + new Argument('[language]', 'The language').choices( + ['all'].concat(LANGUAGES) + ) + ) + .addArgument( + new Argument('[client]', 'The client').choices(['all'].concat(CLIENTS_JS)) + ) + .option('-v, --verbose', 'make the compilation verbose') + .action( + async ( + language: string | undefined, + client: string | undefined, + { verbose } + ) => { + // eslint-disable-next-line no-param-reassign + language = await promptLanguage(language); + // eslint-disable-next-line no-param-reassign + client = await promptClient(client, CLIENTS_JS); + + await buildClients( + generatorList({ language, client, clientList: CLIENTS_JS }), + Boolean(verbose) + ); + } + ); + +program + .command('build:specs') + .description('Build a specified spec') + .addArgument( + new Argument('[client]', 'The client').choices(['all'].concat(CLIENTS)) + ) + .addArgument( + new Argument('[output-format]', 'The output format').choices([ + 'yml', + 'json', + ]) + ) + .option('-v, --verbose', 'make the verification verbose') + .action( + async ( + client: string | undefined, + outputFormat: 'json' | 'yml' | undefined, + { verbose } + ) => { + // eslint-disable-next-line no-param-reassign + client = await promptClient(client); + + if (!outputFormat) { + // eslint-disable-next-line no-param-reassign + ({ outputFormat } = await inquirer.prompt([ + { + type: 'list', + name: 'outputFormat', + message: 'Select the output format', + default: 'yml', + choices: ['yml', 'json'], + }, + ])); + } + + let clientsTodo = [client]; + if (client === 'all') { + clientsTodo = CLIENTS; + } + await buildSpecs(clientsTodo, outputFormat!, Boolean(verbose), true); + } + ); + +program.parse(); diff --git a/scripts/oraLog.ts b/scripts/oraLog.ts new file mode 100644 index 0000000000..412ef0ccf3 --- /dev/null +++ b/scripts/oraLog.ts @@ -0,0 +1,60 @@ +/* eslint-disable no-console */ +import ora from 'ora-classic'; + +class OraLog { + private _text: string; + + constructor(text: string) { + this._text = text; + } + + private maybeText(text?: string): void { + if (text !== undefined) { + this._text = text; + this.start(); + } + } + + start(): this { + console.log(this._text); + return this; + } + + succeed(text?: string): void { + this.maybeText(text); + } + + fail(text?: string): void { + this.maybeText(text); + } + + warn(text?: string): void { + this.maybeText(text); + } + + info(text?: string): void { + this.maybeText(text); + } + + get text(): string { + return this._text; + } + + set text(text: string) { + this._text = text; + this.start(); + } +} + +/** + * Returns a spinner that will log directly in verbose mode, to avoid conflict with other log. + */ +export function createSpinner( + text: string, + verbose: boolean +): ora.Ora | OraLog { + if (verbose) { + return new OraLog(text); + } + return ora(text); +} diff --git a/scripts/package.json b/scripts/package.json index de616795ba..328c4032be 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -5,20 +5,27 @@ "build": "tsc", "createReleaseIssue": "yarn build && node dist/scripts/release/create-release-issue.js", "processRelease": "yarn build && node dist/scripts/release/process-release.js", - "setHostsOptions": "yarn build && node dist/scripts/pre-gen/setHostsOptions.js", "test": "jest" }, "devDependencies": { "@octokit/rest": "18.12.0", + "@types/folder-hash": "^4.0.1", + "@types/inquirer": "8.2.0", "@types/js-yaml": "4.0.5", "@types/node": "16.11.11", + "@types/ora": "^3.2.0", "@types/semver": "7.3.9", + "commander": "9.0.0", "dotenv": "16.0.0", "execa": "5.1.1", + "folder-hash": "^4.0.2", + "inquirer": "8.2.0", "jest": "27.4.7", "js-yaml": "4.1.0", + "ora-classic": "5.4.2", "semver": "7.3.5", "ts-jest": "27.1.3", + "ts-node": "10.5.0", "typescript": "4.5.4" } } diff --git a/scripts/post-gen/global.sh b/scripts/post-gen/global.sh index b8d96b7868..213cd0a169 100755 --- a/scripts/post-gen/global.sh +++ b/scripts/post-gen/global.sh @@ -3,6 +3,10 @@ # Break on non-zero code set -e +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +# Move to the root (easier to locate other scripts) +cd ${DIR}/../.. + LANGUAGE=$1 if [[ $CI ]]; then diff --git a/scripts/post-gen/java.sh b/scripts/post-gen/java.sh index fcf26b8fcf..17f1548a84 100755 --- a/scripts/post-gen/java.sh +++ b/scripts/post-gen/java.sh @@ -3,6 +3,10 @@ # Break on non-zero code set -e +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +# Move to the root (easier to locate other scripts) +cd ${DIR}/../.. + FOLDER=$1 # Restore the oneOf spec diff --git a/scripts/post-gen/javascript.sh b/scripts/post-gen/javascript.sh index 6d0c89335c..1f85a47272 100755 --- a/scripts/post-gen/javascript.sh +++ b/scripts/post-gen/javascript.sh @@ -3,6 +3,11 @@ # Break on non-zero code set -e +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +# Move to the root (easier to locate other scripts) +cd ${DIR}/../.. + + FOLDER=$1 GENERATOR=$2 diff --git a/scripts/post-gen/php.sh b/scripts/post-gen/php.sh index df070ad7e1..7ebcb5aa15 100755 --- a/scripts/post-gen/php.sh +++ b/scripts/post-gen/php.sh @@ -1,5 +1,10 @@ #!/bin/bash set -e + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +# Move to the root (easier to locate other scripts) +cd ${DIR}/../.. + FOLDER=$1 #Move Configuration file diff --git a/scripts/pre-gen/java.sh b/scripts/pre-gen/java.sh index aed3543106..2c5fb11e59 100755 --- a/scripts/pre-gen/java.sh +++ b/scripts/pre-gen/java.sh @@ -3,6 +3,10 @@ # Break on non-zero code set -e +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +# Move to the root (easier to locate other scripts) +cd ${DIR}/../.. + # Remove the oneOf in the spec until it's supported (https://github.com/OpenAPITools/openapi-generator/issues/10880). # Remove the oneOf and only keep the last $ref diff --git a/scripts/pre-gen/setHostsOptions.ts b/scripts/pre-gen/setHostsOptions.ts index 872a5837bf..d7ff3c15c7 100644 --- a/scripts/pre-gen/setHostsOptions.ts +++ b/scripts/pre-gen/setHostsOptions.ts @@ -4,6 +4,8 @@ import { URL } from 'url'; import yaml from 'js-yaml'; +import type { Generator } from '../types'; + type Server = { url: string; variables?: { @@ -31,7 +33,10 @@ type AdditionalProperties = Partial<{ experimentalHost: string; }>; -async function setHostsOptions(): Promise { +export async function setHostsOptions({ + client, + key: generator, +}: Pick): Promise { const openapitoolsPath = path.join(process.cwd(), '../openapitools.json'); if (!(await stat(openapitoolsPath))) { throw new Error( @@ -40,8 +45,6 @@ async function setHostsOptions(): Promise { } const openapitools = JSON.parse(await readFile(openapitoolsPath, 'utf-8')); - const [language, client] = process.argv.slice(2); - const generator = `${language}-${client}`; const generatorOptions = openapitools['generator-cli'].generators[generator]; if (!generator || !generatorOptions) { @@ -115,5 +118,3 @@ async function setHostsOptions(): Promise { throw new Error(`Error reading yaml file ${generator}: ${e}`); } } - -setHostsOptions(); diff --git a/scripts/release/common.ts b/scripts/release/common.ts index 49dc49e23b..61d154dcf0 100644 --- a/scripts/release/common.ts +++ b/scripts/release/common.ts @@ -1,6 +1,3 @@ -import execa from 'execa'; // https://github.com/sindresorhus/execa/tree/v5.1.1 - -import openapitools from '../../openapitools.json'; import config from '../../release.config.json'; export const RELEASED_TAG = config.releasedTag; @@ -8,40 +5,6 @@ export const MAIN_BRANCH = config.mainBranch; export const OWNER = config.owner; export const REPO = config.repo; -export type Run = ( - command: string, - options?: Partial< - execa.SyncOptions & { - errorMessage: string; - } - > -) => execa.ExecaReturnBase['stdout']; - -export const run: Run = ( - command, - { errorMessage = undefined, ...execaOptions } = {} -) => { - let result: execa.ExecaSyncReturnValue; - try { - result = execa.commandSync(command, execaOptions); - } catch (err) { - if (errorMessage) { - throw new Error(`[ERROR] ${errorMessage}`); - } else { - throw err; - } - } - return result.stdout; -}; - -export const LANGS = [ - ...new Set( - Object.keys(openapitools['generator-cli'].generators).map( - (key) => key.split('-')[0] - ) - ), -]; - export function getMarkdownSection(markdown: string, title: string): string { const levelIndicator = title.split(' ')[0]; // e.g. `##` const lines = markdown diff --git a/scripts/types.ts b/scripts/types.ts new file mode 100644 index 0000000000..e5187af650 --- /dev/null +++ b/scripts/types.ts @@ -0,0 +1,7 @@ +export type Generator = Record & { + language: string; + client: string; + key: string; +}; + +export type RunOptions = { errorMessage?: string; verbose?: boolean }; diff --git a/yarn.lock b/yarn.lock index 4dfa561300..7ac52c11cd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3073,6 +3073,13 @@ __metadata: languageName: node linkType: hard +"@types/folder-hash@npm:^4.0.1": + version: 4.0.1 + resolution: "@types/folder-hash@npm:4.0.1" + checksum: df14059d4ba6bde347e078f3d00a2a93965d5e5eb9c9f9c3457b1505fb6bb6fca08083bab20cd7221d2b19c0bcf5a60dc7819720bd8f3ed226b75c3a08dae227 + languageName: node + linkType: hard + "@types/glob@npm:*": version: 7.2.0 resolution: "@types/glob@npm:7.2.0" @@ -3092,6 +3099,16 @@ __metadata: languageName: node linkType: hard +"@types/inquirer@npm:8.2.0": + version: 8.2.0 + resolution: "@types/inquirer@npm:8.2.0" + dependencies: + "@types/through": "*" + rxjs: ^7.2.0 + checksum: bb4b550ca01e892bab9483dfc8f8122dc4eb3291e8b9e1d038fbb6d883335f6ff61692331ae90af452e878c0335b119269ad9909c1ba4b6ec10da6fcf1568011 + languageName: node + linkType: hard + "@types/istanbul-lib-coverage@npm:*, @types/istanbul-lib-coverage@npm:^2.0.0, @types/istanbul-lib-coverage@npm:^2.0.1": version: 2.0.4 resolution: "@types/istanbul-lib-coverage@npm:2.0.4" @@ -3193,6 +3210,15 @@ __metadata: languageName: node linkType: hard +"@types/ora@npm:^3.2.0": + version: 3.2.0 + resolution: "@types/ora@npm:3.2.0" + dependencies: + ora: "*" + checksum: dba266e595627c78aee8e4d2dd7f552c3389f520d27b6474164bf0180a7941c72618b02ef4efcbb02e0a8ae43faf407f9c67ffe35760101fe6b1290bbe46158b + languageName: node + linkType: hard + "@types/parse-json@npm:^4.0.0": version: 4.0.0 resolution: "@types/parse-json@npm:4.0.0" @@ -3240,6 +3266,15 @@ __metadata: languageName: node linkType: hard +"@types/through@npm:*": + version: 0.0.30 + resolution: "@types/through@npm:0.0.30" + dependencies: + "@types/node": "*" + checksum: 9578470db0b527c26e246a1220ae9bffc6bf47f20f89c54aac467c083ab1f7e16c00d9a7b4bb6cb4e2dfae465027270827e5908a6236063f6214625e50585d78 + languageName: node + linkType: hard + "@types/yargs-parser@npm:*": version: 20.2.1 resolution: "@types/yargs-parser@npm:20.2.1" @@ -3568,6 +3603,13 @@ __metadata: languageName: node linkType: hard +"ansi-regex@npm:^6.0.1": + version: 6.0.1 + resolution: "ansi-regex@npm:6.0.1" + checksum: 1ff8b7667cded1de4fa2c9ae283e979fc87036864317da86a2e546725f96406746411d0d85e87a2d12fa5abd715d90006de7fa4fa0477c92321ad3b4c7d4e169 + languageName: node + linkType: hard + "ansi-styles@npm:^3.1.0, ansi-styles@npm:^3.2.1": version: 3.2.1 resolution: "ansi-styles@npm:3.2.1" @@ -3969,6 +4011,17 @@ __metadata: languageName: node linkType: hard +"bl@npm:^5.0.0": + version: 5.0.0 + resolution: "bl@npm:5.0.0" + dependencies: + buffer: ^6.0.3 + inherits: ^2.0.4 + readable-stream: ^3.4.0 + checksum: 5dbbcf9cbcf55221dc21f48968bc8cd6d78faea3c653d496ff8e0c382b95e8b6c4b9e818fe67de2f97ed0cd0c219c350ccce42aca91be33e0ad12e698c615061 + languageName: node + linkType: hard + "boolbase@npm:^1.0.0": version: 1.0.0 resolution: "boolbase@npm:1.0.0" @@ -4002,6 +4055,15 @@ __metadata: languageName: node linkType: hard +"brace-expansion@npm:^2.0.1": + version: 2.0.1 + resolution: "brace-expansion@npm:2.0.1" + dependencies: + balanced-match: ^1.0.0 + checksum: a61e7cd2e8a8505e9f0036b3b6108ba5e926b4b55089eeb5550cd04a471fe216c96d4fe7e4c7f995c728c554ae20ddfc4244cad10aef255e72b62930afd233d1 + languageName: node + linkType: hard + "braces@npm:^3.0.1, braces@npm:~3.0.2": version: 3.0.2 resolution: "braces@npm:3.0.2" @@ -4094,6 +4156,16 @@ __metadata: languageName: node linkType: hard +"buffer@npm:^6.0.3": + version: 6.0.3 + resolution: "buffer@npm:6.0.3" + dependencies: + base64-js: ^1.3.1 + ieee754: ^1.2.1 + checksum: 5ad23293d9a731e4318e420025800b42bf0d264004c0286c8cc010af7a270c7a0f6522e84f54b9ad65cbd6db20b8badbfd8d2ebf4f80fa03dab093b89e68c3f9 + languageName: node + linkType: hard + "builtin-modules@npm:^3.1.0": version: 3.2.0 resolution: "builtin-modules@npm:3.2.0" @@ -4284,6 +4356,13 @@ __metadata: languageName: node linkType: hard +"chalk@npm:^5.0.0": + version: 5.0.0 + resolution: "chalk@npm:5.0.0" + checksum: 6eba7c518b9aa5fe882ae6d14a1ffa58c418d72a3faa7f72af56641f1bbef51b645fca1d6e05d42357b7d3c846cd504c0b7b64d12309cdd07867e3b4411e0d01 + languageName: node + linkType: hard + "char-regex@npm:^1.0.2": version: 1.0.2 resolution: "char-regex@npm:1.0.2" @@ -4391,7 +4470,16 @@ __metadata: languageName: node linkType: hard -"cli-spinners@npm:^2.5.0": +"cli-cursor@npm:^4.0.0": + version: 4.0.0 + resolution: "cli-cursor@npm:4.0.0" + dependencies: + restore-cursor: ^4.0.0 + checksum: ab3f3ea2076e2176a1da29f9d64f72ec3efad51c0960898b56c8a17671365c26e67b735920530eaf7328d61f8bd41c27f46b9cf6e4e10fe2fa44b5e8c0e392cc + languageName: node + linkType: hard + +"cli-spinners@npm:^2.5.0, cli-spinners@npm:^2.6.1": version: 2.6.1 resolution: "cli-spinners@npm:2.6.1" checksum: 423409baaa7a58e5104b46ca1745fbfc5888bbd0b0c5a626e052ae1387060839c8efd512fb127e25769b3dc9562db1dc1b5add6e0b93b7ef64f477feb6416a45 @@ -4529,6 +4617,13 @@ __metadata: languageName: node linkType: hard +"commander@npm:9.0.0": + version: 9.0.0 + resolution: "commander@npm:9.0.0" + checksum: 15066e433d528315ded8261d16bc600d1f3c5671c75021e685ae67e4d62f7551243ff28411b28dc0a6f8b23c2a0f033550ec6f3e66bdf9d11a4fdc2d33dd9802 + languageName: node + linkType: hard + "commander@npm:^2.20.0, commander@npm:^2.7.1": version: 2.20.3 resolution: "commander@npm:2.20.3" @@ -5897,6 +5992,19 @@ __metadata: languageName: node linkType: hard +"folder-hash@npm:^4.0.2": + version: 4.0.2 + resolution: "folder-hash@npm:4.0.2" + dependencies: + debug: ^4.3.3 + graceful-fs: ~4.2.9 + minimatch: ~5.0.0 + bin: + folder-hash: bin/folder-hash + checksum: c087a02f5c39e6869c72e40d196ee642a972e5219b83239ba61f2971c710b6fbee0e9bfbe834cdb59104a7a696596be75c4c3365fc1e7aae8bbe652a650c8092 + languageName: node + linkType: hard + "follow-redirects@npm:^1.14.0, follow-redirects@npm:^1.14.4": version: 1.14.9 resolution: "follow-redirects@npm:1.14.9" @@ -6204,7 +6312,7 @@ __metadata: languageName: node linkType: hard -"graceful-fs@npm:^4.1.6, graceful-fs@npm:^4.2.0, graceful-fs@npm:^4.2.3, graceful-fs@npm:^4.2.6, graceful-fs@npm:^4.2.9": +"graceful-fs@npm:^4.1.6, graceful-fs@npm:^4.2.0, graceful-fs@npm:^4.2.3, graceful-fs@npm:^4.2.6, graceful-fs@npm:^4.2.9, graceful-fs@npm:~4.2.9": version: 4.2.9 resolution: "graceful-fs@npm:4.2.9" checksum: 68ea4e07ff2c041ada184f9278b830375f8e0b75154e3f080af6b70f66172fabb4108d19b3863a96b53fc068a310b9b6493d86d1291acc5f3861eb4b79d26ad6 @@ -6472,7 +6580,7 @@ __metadata: languageName: node linkType: hard -"ieee754@npm:^1.1.13": +"ieee754@npm:^1.1.13, ieee754@npm:^1.2.1": version: 1.2.1 resolution: "ieee754@npm:1.2.1" checksum: 5144c0c9815e54ada181d80a0b810221a253562422e7c6c3a60b1901154184f49326ec239d618c416c1c5945a2e197107aee8d986a3dd836b53dffefd99b5e7e @@ -6746,6 +6854,13 @@ __metadata: languageName: node linkType: hard +"is-interactive@npm:^2.0.0": + version: 2.0.0 + resolution: "is-interactive@npm:2.0.0" + checksum: e8d52ad490bed7ae665032c7675ec07732bbfe25808b0efbc4d5a76b1a1f01c165f332775c63e25e9a03d319ebb6b24f571a9e902669fc1e40b0a60b5be6e26c + languageName: node + linkType: hard + "is-json@npm:^2.0.1": version: 2.0.1 resolution: "is-json@npm:2.0.1" @@ -6860,6 +6975,13 @@ __metadata: languageName: node linkType: hard +"is-unicode-supported@npm:^1.1.0": + version: 1.1.0 + resolution: "is-unicode-supported@npm:1.1.0" + checksum: 1f2504d94383ea180ea25e729b40b1d97398a37325c2e62db96a3e98b457c767259dd5bbf9ab2815e83e5012dc4b61d533e75d12df7f208c470474d821bd5f24 + languageName: node + linkType: hard + "is-weakref@npm:^1.0.1": version: 1.0.2 resolution: "is-weakref@npm:1.0.2" @@ -7898,6 +8020,16 @@ __metadata: languageName: node linkType: hard +"log-symbols@npm:^5.1.0": + version: 5.1.0 + resolution: "log-symbols@npm:5.1.0" + dependencies: + chalk: ^5.0.0 + is-unicode-supported: ^1.1.0 + checksum: 7291b6e7f1b3df6865bdaeb9b59605c832668ac2fa0965c63b1e7dd3700349aec09c1d7d40c368d5041ff58b7f89461a56e4009471921301af7b3609cbff9a29 + languageName: node + linkType: hard + "lru-cache@npm:^6.0.0": version: 6.0.0 resolution: "lru-cache@npm:6.0.0" @@ -8035,6 +8167,15 @@ __metadata: languageName: node linkType: hard +"minimatch@npm:~5.0.0": + version: 5.0.0 + resolution: "minimatch@npm:5.0.0" + dependencies: + brace-expansion: ^2.0.1 + checksum: 810d4165fa2b16d0ffe8eb7586b8b7b7122ab77efa8f351686ffd1e8cbe63997ae9d5e7404fe611a676e59e1a47f7f5e8d9a004e0fe5bb6d5ac35457469116a4 + languageName: node + linkType: hard + "minimist@npm:^1.2.0, minimist@npm:^1.2.3, minimist@npm:^1.2.5": version: 1.2.5 resolution: "minimist@npm:1.2.5" @@ -8634,6 +8775,40 @@ __metadata: languageName: node linkType: hard +"ora-classic@npm:5.4.2": + version: 5.4.2 + resolution: "ora-classic@npm:5.4.2" + dependencies: + bl: ^4.1.0 + chalk: ^4.1.0 + cli-cursor: ^3.1.0 + cli-spinners: ^2.5.0 + is-interactive: ^1.0.0 + is-unicode-supported: ^0.1.0 + log-symbols: ^4.1.0 + strip-ansi: ^6.0.0 + wcwidth: ^1.0.1 + checksum: f52ec97c8fbb051dd883d6bbb75a0b658a173f8230aba0d39b2cf5957b79dcaf7da53dda72d744d726ac8cb7e693fe6d7eb0c27a59298aa35f0ec06651c4626f + languageName: node + linkType: hard + +"ora@npm:*": + version: 6.1.0 + resolution: "ora@npm:6.1.0" + dependencies: + bl: ^5.0.0 + chalk: ^5.0.0 + cli-cursor: ^4.0.0 + cli-spinners: ^2.6.1 + is-interactive: ^2.0.0 + is-unicode-supported: ^1.1.0 + log-symbols: ^5.1.0 + strip-ansi: ^7.0.1 + wcwidth: ^1.0.1 + checksum: 0e84d9c32f2c62617324658ea547963412152f0bf60c1580ed18ec660769713452a6a47d117a1767bb118bb0867720ecc109ef6e12d18fd71ae244683e722b23 + languageName: node + linkType: hard + "ora@npm:^5.4.1": version: 5.4.1 resolution: "ora@npm:5.4.1" @@ -9803,6 +9978,16 @@ __metadata: languageName: node linkType: hard +"restore-cursor@npm:^4.0.0": + version: 4.0.0 + resolution: "restore-cursor@npm:4.0.0" + dependencies: + onetime: ^5.1.0 + signal-exit: ^3.0.2 + checksum: 5b675c5a59763bf26e604289eab35711525f11388d77f409453904e1e69c0d37ae5889295706b2c81d23bd780165084d040f9b68fffc32cc921519031c4fa4af + languageName: node + linkType: hard + "retry@npm:^0.12.0": version: 0.12.0 resolution: "retry@npm:0.12.0" @@ -10002,15 +10187,23 @@ __metadata: resolution: "scripts@workspace:scripts" dependencies: "@octokit/rest": 18.12.0 + "@types/folder-hash": ^4.0.1 + "@types/inquirer": 8.2.0 "@types/js-yaml": 4.0.5 "@types/node": 16.11.11 + "@types/ora": ^3.2.0 "@types/semver": 7.3.9 + commander: 9.0.0 dotenv: 16.0.0 execa: 5.1.1 + folder-hash: ^4.0.2 + inquirer: 8.2.0 jest: 27.4.7 js-yaml: 4.1.0 + ora-classic: 5.4.2 semver: 7.3.5 ts-jest: 27.1.3 + ts-node: 10.5.0 typescript: 4.5.4 languageName: unknown linkType: soft @@ -10386,6 +10579,15 @@ __metadata: languageName: node linkType: hard +"strip-ansi@npm:^7.0.1": + version: 7.0.1 + resolution: "strip-ansi@npm:7.0.1" + dependencies: + ansi-regex: ^6.0.1 + checksum: 257f78fa433520e7f9897722731d78599cb3fce29ff26a20a5e12ba4957463b50a01136f37c43707f4951817a75e90820174853d6ccc240997adc5df8f966039 + languageName: node + linkType: hard + "strip-bom@npm:^3.0.0": version: 3.0.0 resolution: "strip-bom@npm:3.0.0" From 462f626ae2484071835bb332cf6aaf584d4560b3 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Thu, 24 Feb 2022 12:05:43 +0100 Subject: [PATCH 02/20] move cts to scripts --- {tests/src => scripts/cts}/client/generate.ts | 22 ++--- {tests/src => scripts/cts}/client/types.ts | 0 scripts/cts/generate.ts | 39 +++++++++ .../src => scripts/cts}/integrations/.gitkeep | 0 .../cts}/methods/requests/cts.ts | 0 .../cts}/methods/requests/generate.ts | 24 +++--- .../cts}/methods/requests/types.ts | 0 scripts/cts/run.ts | 29 +++++++ {tests/src => scripts/cts}/utils.test.ts | 0 {tests/src => scripts/cts}/utils.ts | 82 ++----------------- tests/CTS/config.json | 10 --- tests/jest.config.ts | 8 -- tests/package.json | 28 ------- tests/src/client/main.ts | 30 ------- tests/src/methods/requests/main.ts | 32 -------- tests/tsconfig.json | 11 --- 16 files changed, 93 insertions(+), 222 deletions(-) rename {tests/src => scripts/cts}/client/generate.ts (90%) rename {tests/src => scripts/cts}/client/types.ts (100%) create mode 100644 scripts/cts/generate.ts rename {tests/src => scripts/cts}/integrations/.gitkeep (100%) rename {tests/src => scripts/cts}/methods/requests/cts.ts (100%) rename {tests/src => scripts/cts}/methods/requests/generate.ts (62%) rename {tests/src => scripts/cts}/methods/requests/types.ts (100%) create mode 100644 scripts/cts/run.ts rename {tests/src => scripts/cts}/utils.test.ts (100%) rename {tests/src => scripts/cts}/utils.ts (55%) delete mode 100644 tests/CTS/config.json delete mode 100644 tests/jest.config.ts delete mode 100644 tests/package.json delete mode 100644 tests/src/client/main.ts delete mode 100644 tests/src/methods/requests/main.ts delete mode 100644 tests/tsconfig.json diff --git a/tests/src/client/generate.ts b/scripts/cts/client/generate.ts similarity index 90% rename from tests/src/client/generate.ts rename to scripts/cts/client/generate.ts index 268c130564..8403f4845c 100644 --- a/tests/src/client/generate.ts +++ b/scripts/cts/client/generate.ts @@ -2,12 +2,11 @@ import fsp from 'fs/promises'; import Mustache from 'mustache'; -import openapitools from '../../../openapitools.json'; +import { exists } from '../../common'; +import type { Generator } from '../../types'; import { walk, - packageNames, createClientName, - exists, createOutputDir, getOutputPath, loadTemplates, @@ -57,10 +56,11 @@ async function loadTests(client: string): Promise { return testsBlocks; } -export async function generateTests( - language: string, - client: string -): Promise { +export async function generateClientTests({ + language, + client, + additionalProperties: { hasRegionalHost, packageName }, +}: Generator): Promise { const testsBlocks = await loadTests(client); if (testsBlocks.length === 0) { @@ -89,14 +89,10 @@ export async function generateTests( const code = Mustache.render( template, { - import: packageNames[language][client], + import: packageName, client: createClientName(client, language), blocks: modifyForMustache(testsBlocks), - hasRegionalHost: openapitools['generator-cli'].generators[ - `${language}-${client}` - ].additionalProperties.hasRegionalHost - ? true - : undefined, + hasRegionalHost: hasRegionalHost ? true : undefined, }, partialTemplates ); diff --git a/tests/src/client/types.ts b/scripts/cts/client/types.ts similarity index 100% rename from tests/src/client/types.ts rename to scripts/cts/client/types.ts diff --git a/scripts/cts/generate.ts b/scripts/cts/generate.ts new file mode 100644 index 0000000000..364fc38490 --- /dev/null +++ b/scripts/cts/generate.ts @@ -0,0 +1,39 @@ +import { getTestOutputFolder } from '../config'; +import { formatter } from '../formatter'; +import { createSpinner } from '../oraLog'; +import type { Generator } from '../types'; + +import { generateRequestsTests } from './methods/requests/generate'; + +async function ctsGenerate( + generator: Generator, + verbose: boolean +): Promise { + const { language, key } = generator; + const spinner = createSpinner(`generating cts for ${key}`, verbose).start(); + switch (language) { + case 'javascript': + await generateRequestsTests(generator); + break; + case 'java': + break; + case 'php': + break; + default: + } + spinner.succeed(); +} + +export async function ctsGenerateMany( + generators: Generator[], + verbose: boolean +): Promise { + for (const gen of generators) { + await ctsGenerate(gen, verbose); + } + + const langs = [...new Set(generators.map((gen) => gen.language))]; + for (const lang of langs) { + await formatter(lang, getTestOutputFolder(lang), verbose); + } +} diff --git a/tests/src/integrations/.gitkeep b/scripts/cts/integrations/.gitkeep similarity index 100% rename from tests/src/integrations/.gitkeep rename to scripts/cts/integrations/.gitkeep diff --git a/tests/src/methods/requests/cts.ts b/scripts/cts/methods/requests/cts.ts similarity index 100% rename from tests/src/methods/requests/cts.ts rename to scripts/cts/methods/requests/cts.ts diff --git a/tests/src/methods/requests/generate.ts b/scripts/cts/methods/requests/generate.ts similarity index 62% rename from tests/src/methods/requests/generate.ts rename to scripts/cts/methods/requests/generate.ts index 4d14c9f4f7..c29d2bfc93 100644 --- a/tests/src/methods/requests/generate.ts +++ b/scripts/cts/methods/requests/generate.ts @@ -2,10 +2,9 @@ import fsp from 'fs/promises'; import Mustache from 'mustache'; -import openapitools from '../../../../openapitools.json'; +import type { Generator } from '../../../types'; import { createClientName, - packageNames, capitalize, getOutputPath, createOutputDir, @@ -16,10 +15,11 @@ import { loadCTS } from './cts'; const testPath = 'methods/requests'; -export async function generateTests( - language: string, - client: string -): Promise { +export async function generateRequestsTests({ + language, + client, + additionalProperties: { hasRegionalHost, packageName }, +}: Generator): Promise { const { requests: template, ...partialTemplates } = await loadTemplates({ language, testPath, @@ -34,21 +34,17 @@ export async function generateTests( const code = Mustache.render( template, { - import: packageNames[language][client], + import: packageName, client: createClientName(client, language), blocks: cts, - hasRegionalHost: openapitools['generator-cli'].generators[ - `${language}-${client}` - ].additionalProperties.hasRegionalHost - ? true - : undefined, + hasRegionalHost: hasRegionalHost ? true : undefined, capitalize() { - return function (text: string, render: (string) => string): string { + return function (text: string, render: (t: string) => string): string { return capitalize(render(text)); }; }, escapeQuotes() { - return function (text: string, render: (string) => string): string { + return function (text: string, render: (t: string) => string): string { return render(text).replace(/"/g, '\\"'); }; }, diff --git a/tests/src/methods/requests/types.ts b/scripts/cts/methods/requests/types.ts similarity index 100% rename from tests/src/methods/requests/types.ts rename to scripts/cts/methods/requests/types.ts diff --git a/scripts/cts/run.ts b/scripts/cts/run.ts new file mode 100644 index 0000000000..861ccd525d --- /dev/null +++ b/scripts/cts/run.ts @@ -0,0 +1,29 @@ +import { CI, run } from '../common'; + +export async function runCts( + language: string, + verbose: boolean +): Promise { + switch (language) { + case 'javascript': + await run('yarn workspace javascript-tests test', { verbose }); + break; + case 'java': + await run('./gradle/gradlew --no-daemon -p tests/output/java test', { + verbose, + }); + break; + case 'php': { + let php = 'php8'; + if (CI) php = 'php'; + + await run( + `${php} ./clients/algoliasearch-client-php/vendor/bin/phpunit ./`, + { verbose } + ); + break; + } + default: + // echo "Skipping unknown language $LANGUAGE to run the CTS" + } +} diff --git a/tests/src/utils.test.ts b/scripts/cts/utils.test.ts similarity index 100% rename from tests/src/utils.test.ts rename to scripts/cts/utils.test.ts diff --git a/tests/src/utils.ts b/scripts/cts/utils.ts similarity index 55% rename from tests/src/utils.ts rename to scripts/cts/utils.ts index a6fd563a2d..29d5743b4a 100644 --- a/tests/src/utils.ts +++ b/scripts/cts/utils.ts @@ -1,30 +1,8 @@ import fsp from 'fs/promises'; import path from 'path'; -import openapitools from '../../openapitools.json'; -import ctsConfig from '../CTS/config.json'; - -// For each generator, we map the packageName with the language and client -export const packageNames: Record< - string, - Record -> = Object.entries(openapitools['generator-cli'].generators).reduce( - (prev, [clientName, clientConfig]) => { - const obj = prev; - const parts = clientName.split('-'); - const lang = parts[0]; - const client = parts.slice(1).join('-'); - - if (!(lang in prev)) { - obj[lang] = {}; - } - - obj[lang][client] = clientConfig.additionalProperties.packageName; - - return obj; - }, - {} as Record> -); +import { exists } from '../common'; +import { getTestExtension, getTestOutputFolder } from '../config'; export async function* walk( dir: string @@ -36,15 +14,6 @@ export async function* walk( } } -export async function exists(filePath: string): Promise { - try { - await fsp.stat(filePath); - return true; - } catch (err) { - return false; - } -} - export function capitalize(str: string): string { return str.charAt(0).toUpperCase() + str.slice(1); } @@ -84,10 +53,6 @@ export function removeObjectName(obj: any): any { return obj; } -export function checkIfLanguageExists(language: string): boolean { - return Boolean(ctsConfig[language]); -} - export function removeEnumType(obj: any): any { if (typeof obj === 'object') { if (Array.isArray(obj)) { @@ -103,43 +68,6 @@ export function removeEnumType(obj: any): any { return obj; } -function printUsage(commandName: string): void { - /* eslint-disable no-console */ - console.log(`usage: ${commandName} language client`); - // eslint-disable-next-line no-process-exit - process.exit(1); -} - -export function parseCLI( - args: string[], - commandName: string -): { lang: string; client: string } { - if (args.length < 3) { - console.log('not enough arguments'); - printUsage(commandName); - } - - const lang = args[2]; - const client = args[3]; - - if (!(lang in packageNames)) { - console.log('Unknown language', lang); - // eslint-disable-next-line no-process-exit - process.exit(1); - } - if (!(client in packageNames[lang])) { - console.log('Unknown client', client); - // eslint-disable-next-line no-process-exit - process.exit(1); - } - /* eslint-enable no-console */ - - return { - lang, - client, - }; -} - export async function createOutputDir({ language, testPath, @@ -148,7 +76,7 @@ export async function createOutputDir({ testPath: string; }): Promise { await fsp.mkdir( - `output/${language}/${ctsConfig[language].outputFolder}/${testPath}`, + `output/${language}/${getTestOutputFolder(language)}/${testPath}`, { recursive: true, } @@ -164,7 +92,9 @@ export function getOutputPath({ client: string; testPath: string; }): string { - return `output/${language}/${ctsConfig[language].outputFolder}/${testPath}/${client}.${ctsConfig[language].extension}`; + return `output/${language}/${getTestOutputFolder( + language + )}/${testPath}/${client}${getTestExtension(language)}`; } export async function loadTemplates({ diff --git a/tests/CTS/config.json b/tests/CTS/config.json deleted file mode 100644 index 0f8a91e9b1..0000000000 --- a/tests/CTS/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "javascript": { - "extension": "test.ts", - "outputFolder": "src" - }, - "java": { - "extension": "test.java", - "outputFolder": "src/test/java/com/algolia" - } -} diff --git a/tests/jest.config.ts b/tests/jest.config.ts deleted file mode 100644 index d2cc09a672..0000000000 --- a/tests/jest.config.ts +++ /dev/null @@ -1,8 +0,0 @@ -import type { Config } from '@jest/types'; - -const config: Config.InitialOptions = { - preset: 'ts-jest', - testEnvironment: 'node', -}; - -export default config; diff --git a/tests/package.json b/tests/package.json deleted file mode 100644 index 5acd34f04a..0000000000 --- a/tests/package.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "name": "tests", - "version": "1.0.0", - "workspaces": [ - "output/javascript" - ], - "scripts": { - "build": "tsc", - "generate": "yarn generate:methods:requets ${0:-javascript} ${1:-search} && yarn generate:client ${0:-javascript} ${1:-search}", - "generate:methods:requets": "node dist/tests/src/methods/requests/main.js ${0:-javascript} ${1:-search}", - "generate:client": "node dist/tests/src/client/main.js ${0:-javascript} ${1:-search}", - "format": "../scripts/formatter.sh ${0:-javascript} tests/output/${0:-javascript}/src && yarn lint", - "lint": "eslint --ext=ts ./src", - "test:scripts": "jest" - }, - "devDependencies": { - "@apidevtools/swagger-parser": "10.0.3", - "@types/jest": "27.4.0", - "@types/mustache": "4.1.2", - "@types/node": "16.11.11", - "eslint": "8.6.0", - "jest": "27.4.7", - "mustache": "4.2.0", - "openapi-types": "10.0.0", - "ts-jest": "27.1.3", - "typescript": "4.5.4" - } -} diff --git a/tests/src/client/main.ts b/tests/src/client/main.ts deleted file mode 100644 index 0d4476ffa3..0000000000 --- a/tests/src/client/main.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { parseCLI, checkIfLanguageExists } from '../utils'; - -import { generateTests } from './generate'; - -async function main(): Promise { - const { lang, client } = parseCLI(process.argv, 'generate:client'); - - if (!checkIfLanguageExists(lang)) { - // eslint-disable-next-line no-console - console.log( - `Skipping CTS generation > generate:client for ${lang}-${client}: Language not present in the config.json file` - ); - - return; - } - - // eslint-disable-next-line no-console - console.log(`Generating CTS > generate:client for ${lang}-${client}`); - - try { - await generateTests(lang, client); - } catch (e) { - if (e instanceof Error) { - // eslint-disable-next-line no-console - console.error(e); - } - } -} - -main(); diff --git a/tests/src/methods/requests/main.ts b/tests/src/methods/requests/main.ts deleted file mode 100644 index 9a5fd15ecd..0000000000 --- a/tests/src/methods/requests/main.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { parseCLI, checkIfLanguageExists } from '../../utils'; - -import { generateTests } from './generate'; - -async function main(): Promise { - const { lang, client } = parseCLI(process.argv, 'generate:methods:requests'); - - if (!checkIfLanguageExists(lang)) { - // eslint-disable-next-line no-console - console.log( - `Skipping CTS generation > generate:methods:requests for ${lang}-${client}: Language not present in the config.json file` - ); - - return; - } - - // eslint-disable-next-line no-console - console.log( - `Generating CTS > generate:methods:requests for ${lang}-${client}` - ); - - try { - await generateTests(lang, client); - } catch (e) { - if (e instanceof Error) { - // eslint-disable-next-line no-console - console.error(e); - } - } -} - -main(); diff --git a/tests/tsconfig.json b/tests/tsconfig.json deleted file mode 100644 index a9efb8ab01..0000000000 --- a/tests/tsconfig.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "extends": "../base.tsconfig.json", - "compilerOptions": { - "typeRoots": ["../node_modules/@types"], - "types": ["node", "jest"], - "lib": ["ESNext", "dom", "dom.iterable"], - "outDir": "dist" - }, - "include": ["src"], - "exclude": ["dist", "output"] -} From 2f023999fe3688922f32f1481591ef57785f42c3 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Thu, 24 Feb 2022 14:48:01 +0100 Subject: [PATCH 03/20] convert all scripts --- clients.config.json | 12 ++- openapitools.json | 3 +- package.json | 10 +-- scripts/buildSpecs.ts | 6 +- scripts/common.ts | 30 ++++--- scripts/config.ts | 13 +++ scripts/cts/client/generate.ts | 34 ++++---- scripts/cts/generate.ts | 13 ++- scripts/cts/methods/requests/cts.ts | 9 +- scripts/cts/methods/requests/generate.ts | 20 +++-- scripts/cts/utils.test.ts | 11 --- scripts/cts/utils.ts | 18 ++-- scripts/formatter.ts | 5 +- scripts/generate.ts | 6 +- scripts/index.ts | 83 +++++++++++++++--- scripts/multiplexer.sh | 102 ---------------------- scripts/package.json | 12 ++- scripts/playground.sh | 28 ------ scripts/playground.ts | 32 +++++++ scripts/runCTS.sh | 31 ------- scripts/types.ts | 6 +- yarn.lock | 104 +++-------------------- 22 files changed, 244 insertions(+), 344 deletions(-) create mode 100644 scripts/config.ts delete mode 100755 scripts/multiplexer.sh delete mode 100755 scripts/playground.sh create mode 100644 scripts/playground.ts delete mode 100755 scripts/runCTS.sh diff --git a/clients.config.json b/clients.config.json index 9cf7d23988..a8bf220c37 100644 --- a/clients.config.json +++ b/clients.config.json @@ -1,8 +1,16 @@ { "java": { - "folder": "clients/algoliasearch-client-java-2" + "folder": "clients/algoliasearch-client-java-2", + "tests": { + "extension": ".test.ts", + "outputFolder": "src" + } }, "javascript": { - "folder": "clients/algoliasearch-client-javascript" + "folder": "clients/algoliasearch-client-javascript", + "texts": { + "extension": ".test.java", + "outputFolder": "src/test/java/com/algolia" + } } } diff --git a/openapitools.json b/openapitools.json index 74a71578b0..491b4a390a 100644 --- a/openapitools.json +++ b/openapitools.json @@ -241,7 +241,8 @@ "sourceFolder": "algoliasearch-core", "java8": true, "dateLibrary": "java8", - "packageName": "algoliasearch-client-java-2" + "packageName": "algoliasearch-client-java-2", + "packageVersion": "0.0.1" } }, "php-search": { diff --git a/package.json b/package.json index 994aebfcaa..80fdc23814 100644 --- a/package.json +++ b/package.json @@ -11,18 +11,14 @@ "scripts": { "api": "yarn workspace scripts ts-node ./index.ts", "clean": "rm -rf **/dist **/build **/node_modules **/.gradle", - "cts:generate": "yarn workspace tests build && ./scripts/multiplexer.sh ${2:-nonverbose} yarn workspace tests generate ${0:-all} ${1:-all}", - "cts:test": "./scripts/multiplexer.sh ${1:-nonverbose} ./scripts/runCTS.sh ${0:-all} all", - "cts:test:scripts": "yarn workspace tests test:scripts", - "cts:lint:scripts": "eslint --ext=ts tests/src/", + "cts:test:scripts": "yarn workspace scripts test", + "cts:lint:scripts": "eslint --ext=ts scripts/", "docker:build": "./scripts/docker/build.sh", "docker:clean": "docker stop dev; docker rm -f dev; docker image rm -f api-clients-automation", "docker:mount": "./scripts/docker/mount.sh", "docker:setup": "yarn docker:clean && yarn docker:build && yarn docker:mount", - "docker": "docker exec -it dev yarn $*", - "lint": "eslint --ext=ts .", + "docker": "docker exec -it dev yarn api $*", "playground:browser": "yarn workspace javascript-browser-playground start", - "playground": "yarn && ./scripts/multiplexer.sh ${2:-nonverbose} ./scripts/playground.sh ${0:-javascript} ${1:-search}", "specs:fix": "eslint --ext=yml specs/ --fix", "specs:lint": "eslint --ext=yml specs/$0", "github-actions:lint": "eslint --ext=yml .github/", diff --git a/scripts/buildSpecs.ts b/scripts/buildSpecs.ts index a78787db90..bc9370da06 100644 --- a/scripts/buildSpecs.ts +++ b/scripts/buildSpecs.ts @@ -2,7 +2,7 @@ import fsp from 'fs/promises'; import { hashElement } from 'folder-hash'; -import { fileExists, run } from './common'; +import { exists, run } from './common'; import { createSpinner } from './oraLog'; async function buildSpec( @@ -18,10 +18,10 @@ async function buildSpec( `checking cache for ${client}`, verbose ).start(); - if (await fileExists(`../specs/bundled/${client}.yml`)) { + if (await exists(`../specs/bundled/${client}.yml`)) { const hash = (await hashElement(`../specs/${client}`)).hash; // compare with stored cache - if (await fileExists(cacheFile)) { + if (await exists(cacheFile)) { const storedHash = (await fsp.readFile(cacheFile)).toString(); if (storedHash === hash) { spinner.succeed( diff --git a/scripts/common.ts b/scripts/common.ts index fd52b372a2..7cbe05e805 100644 --- a/scripts/common.ts +++ b/scripts/common.ts @@ -1,8 +1,8 @@ import fsp from 'fs/promises'; +import path from 'path'; import execa from 'execa'; // https://github.com/sindresorhus/execa/tree/v5.1.1 -import clientsConfig from '../clients.config.json'; import openapitools from '../openapitools.json'; import type { Generator, RunOptions } from './types'; @@ -10,6 +10,9 @@ import type { Generator, RunOptions } from './types'; export const CI = Boolean(process.env.CI); export const DOCKER = Boolean(process.env.DOCKER); +// This script is run by `yarn workspace ...`, which means the current working directory is `./script` +export const ROOT_DIR = path.resolve(process.cwd(), '..'); + export const GENERATORS = Object.fromEntries( Object.entries(openapitools['generator-cli'].generators).map(([key, gen]) => { return [key, { ...gen, ...splitGeneratorKey(key) }]; @@ -39,13 +42,9 @@ export function createGeneratorKey({ return `${language}-${client}`; } -export function getLanguageFolder(language: string): string { - return clientsConfig[language].folder; -} - export async function run( command: string, - { errorMessage, verbose }: RunOptions = {} + { errorMessage, verbose, cwd = ROOT_DIR }: RunOptions = {} ): Promise { try { if (verbose) { @@ -53,11 +52,11 @@ export async function run( await execa.command(command, { stdout: 'inherit', shell: 'bash', - cwd: '../', + cwd, }) ).stdout; } - return (await execa.command(command, { shell: 'bash', cwd: '../' })).stdout; + return (await execa.command(command, { shell: 'bash', cwd })).stdout; } catch (err) { if (errorMessage) { throw new Error(`[ERROR] ${errorMessage}`); @@ -67,21 +66,26 @@ export async function run( } } -export async function fileExists(path: string): Promise { +export async function exists(ppath: string): Promise { try { - return (await fsp.stat(path)).isFile(); + await fsp.stat(ppath); + return true; } catch { return false; } } +export function getAbsolutePath(ppath: string): string { + return path.resolve(ROOT_DIR, ppath); +} + export async function runIfExists( - script: string, + scriptFile: string, args: string, opts: RunOptions = {} ): Promise { - if (await fileExists(`../${script}`)) { - return await run(`${script} ${args}`, opts); + if (await exists(getAbsolutePath(scriptFile))) { + return await run(`${scriptFile} ${args}`, opts); } return ''; } diff --git a/scripts/config.ts b/scripts/config.ts new file mode 100644 index 0000000000..c83a43911f --- /dev/null +++ b/scripts/config.ts @@ -0,0 +1,13 @@ +import clientsConfig from '../clients.config.json'; + +export function getLanguageFolder(language: string): string { + return clientsConfig[language].folder; +} + +export function getTestExtension(language: string): string { + return clientsConfig[language].tests.extension; +} + +export function getTestOutputFolder(language: string): string { + return clientsConfig[language].tests.outputFolder; +} diff --git a/scripts/cts/client/generate.ts b/scripts/cts/client/generate.ts index 8403f4845c..f4cc76d1c3 100644 --- a/scripts/cts/client/generate.ts +++ b/scripts/cts/client/generate.ts @@ -2,7 +2,8 @@ import fsp from 'fs/promises'; import Mustache from 'mustache'; -import { exists } from '../../common'; +import { exists, getAbsolutePath } from '../../common'; +import { createSpinner } from '../../oraLog'; import type { Generator } from '../../types'; import { walk, @@ -18,7 +19,7 @@ const testPath = 'client'; async function loadTests(client: string): Promise { const testsBlocks: TestsBlock[] = []; - const clientPath = `./CTS/client/${client}`; + const clientPath = getAbsolutePath(`tests/CTS/client/${client}`); if (!(await exists(clientPath))) { return []; @@ -56,20 +57,23 @@ async function loadTests(client: string): Promise { return testsBlocks; } -export async function generateClientTests({ - language, - client, - additionalProperties: { hasRegionalHost, packageName }, -}: Generator): Promise { +export async function generateClientTests( + { + language, + client, + additionalProperties: { hasRegionalHost, packageName }, + }: Generator, + verbose: boolean +): Promise { + let spinner = createSpinner('generating client tests', verbose).start(); const testsBlocks = await loadTests(client); if (testsBlocks.length === 0) { - // eslint-disable-next-line no-console - console.warn( - `Skipping because tests dont't exist for CTS > generate:client for ${language}-${client}` - ); + spinner.warn("Skipping because tests doesn' exist"); return; } + spinner.info(); + spinner = createSpinner('loading templates', verbose).start(); await createOutputDir({ language, testPath }); @@ -79,13 +83,11 @@ export async function generateClientTests({ }); if (!template) { - // eslint-disable-next-line no-console - console.warn( - `Skipping because template doesn't exist for CTS > generate:client for ${language}-${client}` - ); + spinner.warn("Skipping because template doesn't exist"); return; } + spinner.text = 'rendering templates'; const code = Mustache.render( template, { @@ -97,6 +99,8 @@ export async function generateClientTests({ partialTemplates ); await fsp.writeFile(getOutputPath({ language, client, testPath }), code); + + spinner.succeed(); } function serializeParameters(parameters: any): string { diff --git a/scripts/cts/generate.ts b/scripts/cts/generate.ts index 364fc38490..b1349a027c 100644 --- a/scripts/cts/generate.ts +++ b/scripts/cts/generate.ts @@ -3,25 +3,22 @@ import { formatter } from '../formatter'; import { createSpinner } from '../oraLog'; import type { Generator } from '../types'; +import { generateClientTests } from './client/generate'; import { generateRequestsTests } from './methods/requests/generate'; async function ctsGenerate( generator: Generator, verbose: boolean ): Promise { - const { language, key } = generator; - const spinner = createSpinner(`generating cts for ${key}`, verbose).start(); - switch (language) { + createSpinner(`generating CTS for ${generator.key}`, verbose).start().info(); + switch (generator.language) { case 'javascript': - await generateRequestsTests(generator); - break; case 'java': - break; - case 'php': + await generateRequestsTests(generator, verbose); + await generateClientTests(generator, verbose); break; default: } - spinner.succeed(); } export async function ctsGenerateMany( diff --git a/scripts/cts/methods/requests/cts.ts b/scripts/cts/methods/requests/cts.ts index 0b79e23e70..0214c62c58 100644 --- a/scripts/cts/methods/requests/cts.ts +++ b/scripts/cts/methods/requests/cts.ts @@ -3,6 +3,7 @@ import fsp from 'fs/promises'; import SwaggerParser from '@apidevtools/swagger-parser'; import type { OpenAPIV3 } from 'openapi-types'; +import { getAbsolutePath } from '../../../common'; import { removeEnumType, removeObjectName, walk } from '../../utils'; import type { @@ -143,7 +144,9 @@ function createParamWithDataType({ async function loadRequestsCTS(client: string): Promise { // load the list of operations from the spec - const spec = await SwaggerParser.validate(`../specs/${client}/spec.yml`); + const spec = await SwaggerParser.validate( + getAbsolutePath(`specs/${client}/spec.yml`) + ); if (!spec.paths) { throw new Error(`No paths found for spec ${client}/spec.yml`); } @@ -154,7 +157,9 @@ async function loadRequestsCTS(client: string): Promise { const ctsClient: CTSBlock[] = []; - for await (const file of walk(`./CTS/methods/requests/${client}`)) { + for await (const file of walk( + getAbsolutePath(`tests/CTS/methods/requests/${client}`) + )) { if (!file.name.endsWith('json')) { continue; } diff --git a/scripts/cts/methods/requests/generate.ts b/scripts/cts/methods/requests/generate.ts index c29d2bfc93..dd55d7383e 100644 --- a/scripts/cts/methods/requests/generate.ts +++ b/scripts/cts/methods/requests/generate.ts @@ -2,6 +2,7 @@ import fsp from 'fs/promises'; import Mustache from 'mustache'; +import { createSpinner } from '../../../oraLog'; import type { Generator } from '../../../types'; import { createClientName, @@ -15,15 +16,22 @@ import { loadCTS } from './cts'; const testPath = 'methods/requests'; -export async function generateRequestsTests({ - language, - client, - additionalProperties: { hasRegionalHost, packageName }, -}: Generator): Promise { +export async function generateRequestsTests( + { + language, + client, + additionalProperties: { hasRegionalHost, packageName }, + }: Generator, + verbose: boolean +): Promise { + createSpinner('generating requests tests', verbose).start().info(); + const spinner = createSpinner('loading templates', verbose).start(); const { requests: template, ...partialTemplates } = await loadTemplates({ language, testPath, }); + + spinner.text = 'loading CTS'; const cts = (await loadCTS(client)).requests; await createOutputDir({ language, testPath }); @@ -31,6 +39,7 @@ export async function generateRequestsTests({ return; } + spinner.text = 'rendering templates'; const code = Mustache.render( template, { @@ -53,4 +62,5 @@ export async function generateRequestsTests({ ); await fsp.writeFile(getOutputPath({ language, client, testPath }), code); + spinner.succeed(); } diff --git a/scripts/cts/utils.test.ts b/scripts/cts/utils.test.ts index de0f60ea20..dc1401bb46 100644 --- a/scripts/cts/utils.test.ts +++ b/scripts/cts/utils.test.ts @@ -1,6 +1,5 @@ import { capitalize, - checkIfLanguageExists, createClientName, removeEnumType, removeObjectName, @@ -116,14 +115,4 @@ describe('utils', () => { }); }); }); - - describe('checkIfLanguageExists', () => { - it('returns `true` if the language is present in the config', () => { - expect(checkIfLanguageExists('javascript')).toBe(true); - }); - - it('returns `false` if the language is not present in the config', () => { - expect(checkIfLanguageExists('algo')).toBe(false); - }); - }); }); diff --git a/scripts/cts/utils.ts b/scripts/cts/utils.ts index 29d5743b4a..e4298f01a1 100644 --- a/scripts/cts/utils.ts +++ b/scripts/cts/utils.ts @@ -1,7 +1,7 @@ import fsp from 'fs/promises'; import path from 'path'; -import { exists } from '../common'; +import { exists, getAbsolutePath } from '../common'; import { getTestExtension, getTestOutputFolder } from '../config'; export async function* walk( @@ -76,7 +76,9 @@ export async function createOutputDir({ testPath: string; }): Promise { await fsp.mkdir( - `output/${language}/${getTestOutputFolder(language)}/${testPath}`, + getAbsolutePath( + `tests/output/${language}/${getTestOutputFolder(language)}/${testPath}` + ), { recursive: true, } @@ -92,9 +94,11 @@ export function getOutputPath({ client: string; testPath: string; }): string { - return `output/${language}/${getTestOutputFolder( - language - )}/${testPath}/${client}${getTestExtension(language)}`; + return getAbsolutePath( + `tests/output/${language}/${getTestOutputFolder( + language + )}/${testPath}/${client}${getTestExtension(language)}` + ); } export async function loadTemplates({ @@ -105,7 +109,9 @@ export async function loadTemplates({ testPath: string; }): Promise> { const templates: Record = {}; - const templatePath = `./CTS/${testPath}/templates/${language}`; + const templatePath = getAbsolutePath( + `./CTS/${testPath}/templates/${language}` + ); if (!(await exists(templatePath))) { return {}; diff --git a/scripts/formatter.ts b/scripts/formatter.ts index ed16a7e338..55a543433c 100644 --- a/scripts/formatter.ts +++ b/scripts/formatter.ts @@ -1,10 +1,12 @@ import { CI, run } from './common'; +import { createSpinner } from './oraLog'; export async function formatter( language: string, folder: string, - verbose?: boolean + verbose = false ): Promise { + const spinner = createSpinner(`formatting ${language}`, verbose).start(); let cmd = ''; switch (language) { case 'javascript': @@ -30,4 +32,5 @@ export async function formatter( return; } await run(cmd, { verbose }); + spinner.succeed(); } diff --git a/scripts/generate.ts b/scripts/generate.ts index 965a669d3d..3ca1bf890d 100644 --- a/scripts/generate.ts +++ b/scripts/generate.ts @@ -1,5 +1,6 @@ import { buildSpecs } from './buildSpecs'; -import { CI, getLanguageFolder, run, runIfExists } from './common'; +import { CI, run, runIfExists } from './common'; +import { getLanguageFolder } from './config'; import { formatter } from './formatter'; import { createSpinner } from './oraLog'; import { setHostsOptions } from './pre-gen/setHostsOptions'; @@ -60,9 +61,6 @@ export async function generate( const langs = [...new Set(generators.map((gen) => gen.language))]; for (const lang of langs) { - const spinner = createSpinner(`formatting ${lang}`, verbose).start(); await formatter(lang, getLanguageFolder(lang), verbose); - - spinner.succeed(); } } diff --git a/scripts/index.ts b/scripts/index.ts index 2a8d2b778e..7b8bd7ccaa 100644 --- a/scripts/index.ts +++ b/scripts/index.ts @@ -1,3 +1,4 @@ +/* eslint-disable no-param-reassign */ import { Argument, program } from 'commander'; import inquirer from 'inquirer'; @@ -12,7 +13,10 @@ import { GENERATORS, LANGUAGES, } from './common'; +import { ctsGenerateMany } from './cts/generate'; +import { runCts } from './cts/run'; import { generate } from './generate'; +import { playground } from './playground'; import type { Generator } from './types'; if (!CI && !DOCKER) { @@ -102,17 +106,17 @@ program client: string | undefined, { verbose } ) => { - // eslint-disable-next-line no-param-reassign language = await promptLanguage(language); - // eslint-disable-next-line no-param-reassign client = await promptClient(client); await generate(generatorList({ language, client }), Boolean(verbose)); } ); -program - .command('build:clients') +const buildCommand = program.command('build'); + +buildCommand + .command('clients') .description('Build a specified client') .addArgument( new Argument('[language]', 'The language').choices( @@ -129,9 +133,7 @@ program client: string | undefined, { verbose } ) => { - // eslint-disable-next-line no-param-reassign language = await promptLanguage(language); - // eslint-disable-next-line no-param-reassign client = await promptClient(client, CLIENTS_JS); await buildClients( @@ -141,8 +143,8 @@ program } ); -program - .command('build:specs') +buildCommand + .command('specs') .description('Build a specified spec') .addArgument( new Argument('[client]', 'The client').choices(['all'].concat(CLIENTS)) @@ -160,11 +162,9 @@ program outputFormat: 'json' | 'yml' | undefined, { verbose } ) => { - // eslint-disable-next-line no-param-reassign client = await promptClient(client); if (!outputFormat) { - // eslint-disable-next-line no-param-reassign ({ outputFormat } = await inquirer.prompt([ { type: 'list', @@ -184,4 +184,67 @@ program } ); +const ctsCommand = program.command('cts'); + +ctsCommand + .command('generate') + .description('Generate the CTS tests') + .addArgument( + new Argument('[language]', 'The language').choices( + ['all'].concat(LANGUAGES) + ) + ) + .addArgument( + new Argument('[client]', 'The client').choices(['all'].concat(CLIENTS)) + ) + .option('-v, --verbose', 'make the generation verbose') + .action( + async ( + language: string | undefined, + client: string | undefined, + { verbose } + ) => { + language = await promptLanguage(language); + client = await promptClient(client); + + await ctsGenerateMany( + generatorList({ language, client }), + Boolean(verbose) + ); + } + ); + +ctsCommand + .command('run') + .description('Run the tests for the CTS') + .addArgument( + new Argument('[language]', 'The language').choices( + ['all'].concat(LANGUAGES) + ) + ) + .option('-v, --verbose', 'make the generation verbose') + .action(async (language: string | undefined, { verbose }) => { + language = await promptLanguage(language); + + await runCts(language, Boolean(verbose)); + }); + +program + .command('playground') + .description('Run the playground') + .addArgument(new Argument('[language]', 'The language').choices(LANGUAGES)) + .addArgument( + new Argument('[client]', 'The client').choices(['all'].concat(CLIENTS)) + ) + .action(async (language: string | undefined, client: string | undefined) => { + language = await promptLanguage(language); + client = await promptClient(client); + + await playground({ + language, + client, + key: createGeneratorKey({ language, client }), + }); + }); + program.parse(); diff --git a/scripts/multiplexer.sh b/scripts/multiplexer.sh deleted file mode 100755 index 48cc0b561b..0000000000 --- a/scripts/multiplexer.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/bash -# Call this script with multiplexer.sh -# to run the cmd for all the required lang-client combination - -if [[ ! $CI ]] && [[ ! $DOCKER ]]; then - echo "You should run scripts via the docker container, see README.md" - - exit 1 -fi - -# Break on non-zero code -set -e - -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" -# Move to the root (easier to locate other scripts) -cd ${DIR}/.. - -CMD=${@:2:$#-3} # all arguments but the last 2 and first one -LANGUAGE=${@: -2:1} # before last argument -CLIENT=${@: -1} # last argument - -if [[ $1 == "verbose" ]]; then - export VERBOSE=true - echo "Verbose mode" -fi - -if [[ $CMD == "./scripts/playground.sh" ]] && ([[ $LANGUAGE == "all" ]] || [[ $CLIENT == "all" ]]); then - echo "You cannot use 'all' when running the playground, please specify the language and client" - - exit 1 -fi - -if [[ $CMD == "./scripts/runCTS.sh" ]]; then - if [[ $CLIENT == "all" ]]; then - CLIENT=search # dummy client to only run once - else - echo "You must use 'all' clients when testing the CTS, as they all run at the same time" - - exit 1 - fi -fi - -LANGUAGES=() -CLIENTS=() - -GENERATORS=() - -find_clients_and_languages() { - echo "> Searching for available languages and clients..." - - GENERATORS=( $(cat openapitools.json | jq '."generator-cli".generators' | jq -r 'keys[]') ) - - for generator in "${GENERATORS[@]}"; do - local lang=${generator%%-*} - local client=${generator#*-} - - if [[ ! ${LANGUAGES[*]} =~ $lang ]]; then - LANGUAGES+=($lang) - fi - - if [[ ! ${CLIENTS[*]} =~ $client ]]; then - CLIENTS+=($client) - fi - done -} - -find_clients_and_languages - -if [[ $LANGUAGE == "all" ]]; then - LANGUAGE=("${LANGUAGES[@]}") -elif [[ " ${LANGUAGES[*]} " =~ " ${LANGUAGE} " ]]; then - LANGUAGE=($LANGUAGE) -else - echo "Unknown language ${LANGUAGE}" - exit 1 -fi - -if [[ $CLIENT == "all" ]]; then - CLIENT=("${CLIENTS[@]}") -elif [[ " ${CLIENTS[*]} " =~ " ${CLIENT} " ]]; then - CLIENT=($CLIENT) -else - echo "Unknown client ${CLIENT}" - exit 1 -fi - -for lang in "${LANGUAGE[@]}"; do - for client in "${CLIENT[@]}"; do - if [[ " ${GENERATORS[*]} " =~ " ${lang}-${client} " ]]; then - $CMD $lang $client - echo "" - fi - done -done - -# Format after every client for the CTS -if [[ $CMD == 'yarn workspace tests generate' ]]; then - for lang in "${LANGUAGE[@]}"; do - yarn workspace tests format $lang - done - yarn cts:lint:scripts --fix -fi diff --git a/scripts/package.json b/scripts/package.json index 328c4032be..deb90a4d6e 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -8,20 +8,26 @@ "test": "jest" }, "devDependencies": { + "@apidevtools/swagger-parser": "10.0.3", "@octokit/rest": "18.12.0", - "@types/folder-hash": "^4.0.1", + "@types/folder-hash": "4.0.1", "@types/inquirer": "8.2.0", + "@types/jest": "27.4.0", "@types/js-yaml": "4.0.5", + "@types/mustache": "4.1.2", "@types/node": "16.11.11", - "@types/ora": "^3.2.0", + "@types/ora": "3.2.0", "@types/semver": "7.3.9", "commander": "9.0.0", "dotenv": "16.0.0", + "eslint": "8.6.0", "execa": "5.1.1", - "folder-hash": "^4.0.2", + "folder-hash": "4.0.2", "inquirer": "8.2.0", "jest": "27.4.7", "js-yaml": "4.1.0", + "mustache": "4.2.0", + "openapi-types": "10.0.0", "ora-classic": "5.4.2", "semver": "7.3.5", "ts-jest": "27.1.3", diff --git a/scripts/playground.sh b/scripts/playground.sh deleted file mode 100755 index e422a0ee1b..0000000000 --- a/scripts/playground.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -# Break on non-zero code -set -e - -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" -# Move to the root (easier to locate other scripts) -cd ${DIR}/.. - -LANGUAGE=$1 -CLIENT=$2 - -# Run the pre generation script if it exists. -run_playground() { - if [[ $LANGUAGE == 'javascript' ]]; then - yarn workspace javascript-playground start:$CLIENT - elif [[ $LANGUAGE == 'java' ]]; then - ./gradle/gradlew --no-daemon -p playground/java run - elif [[ $lang == 'php' ]]; then - cd playground/php - composer update - composer dump-autoload - cd src - php8 $client.php - fi -} - -run_playground diff --git a/scripts/playground.ts b/scripts/playground.ts new file mode 100644 index 0000000000..da7fe28a6f --- /dev/null +++ b/scripts/playground.ts @@ -0,0 +1,32 @@ +import { run } from './common'; +import type { Generator } from './types'; + +export async function playground({ + language, + client, +}: Generator): Promise { + const verbose = true; + switch (language) { + case 'javascript': + await run(`yarn workspace javascript-playground start:${client}`, { + verbose, + }); + break; + case 'java': + await run(`./gradle/gradlew --no-daemon -p playground/java run`, { + verbose, + }); + break; + case 'php': + await run( + `cd playground/php && \ + composer update && \ + composer dump-autoload && \ + cd src && \ + php8 ${client}.php`, + { verbose } + ); + break; + default: + } +} diff --git a/scripts/runCTS.sh b/scripts/runCTS.sh deleted file mode 100755 index 28fa3ab056..0000000000 --- a/scripts/runCTS.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# Break on non-zero code -set -e - -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" -# Move to the root (easier to locate other scripts) -cd ${DIR}/.. - -LANGUAGE=$1 - -# Run the pre generation script if it exists. -run_cts() { - if [[ $LANGUAGE == 'javascript' ]]; then - yarn workspace javascript-tests test - elif [[ $LANGUAGE == 'php' ]]; then - if [[ $CI ]]; then - PHP="php" - else - PHP="php8" - fi - $PHP ./clients/algoliasearch-client-php/vendor/bin/phpunit ./ - elif [[ $LANGUAGE == 'java' ]]; then - ./gradle/gradlew --no-daemon -p tests/output/java test - else - echo "Skipping unknown language $LANGUAGE to run the CTS" - exit 0 - fi -} - -run_cts diff --git a/scripts/types.ts b/scripts/types.ts index e5187af650..c2dcba8f55 100644 --- a/scripts/types.ts +++ b/scripts/types.ts @@ -4,4 +4,8 @@ export type Generator = Record & { key: string; }; -export type RunOptions = { errorMessage?: string; verbose?: boolean }; +export type RunOptions = { + errorMessage?: string; + verbose?: boolean; + cwd?: string; +}; diff --git a/yarn.lock b/yarn.lock index 7ac52c11cd..eb5bda0126 100644 --- a/yarn.lock +++ b/yarn.lock @@ -65,7 +65,7 @@ __metadata: languageName: unknown linkType: soft -"@algolia/client-insights@5.0.0, @algolia/client-insights@workspace:clients/algoliasearch-client-javascript/packages/client-insights": +"@algolia/client-insights@workspace:clients/algoliasearch-client-javascript/packages/client-insights": version: 0.0.0-use.local resolution: "@algolia/client-insights@workspace:clients/algoliasearch-client-javascript/packages/client-insights" dependencies: @@ -3073,7 +3073,7 @@ __metadata: languageName: node linkType: hard -"@types/folder-hash@npm:^4.0.1": +"@types/folder-hash@npm:4.0.1": version: 4.0.1 resolution: "@types/folder-hash@npm:4.0.1" checksum: df14059d4ba6bde347e078f3d00a2a93965d5e5eb9c9f9c3457b1505fb6bb6fca08083bab20cd7221d2b19c0bcf5a60dc7819720bd8f3ed226b75c3a08dae227 @@ -3134,16 +3134,6 @@ __metadata: languageName: node linkType: hard -"@types/jest@npm:27.0.3": - version: 27.0.3 - resolution: "@types/jest@npm:27.0.3" - dependencies: - jest-diff: ^27.0.0 - pretty-format: ^27.0.0 - checksum: 3683a9945821966f6dccddf337219a5d682633687c9d30df859223db553589f63e9b2c34e69f0cc845c86ffcf115742f25c12ea03c8d33d2244890fdc0af61e2 - languageName: node - linkType: hard - "@types/jest@npm:27.4.0": version: 27.4.0 resolution: "@types/jest@npm:27.4.0" @@ -3210,7 +3200,7 @@ __metadata: languageName: node linkType: hard -"@types/ora@npm:^3.2.0": +"@types/ora@npm:3.2.0": version: 3.2.0 resolution: "@types/ora@npm:3.2.0" dependencies: @@ -5992,7 +5982,7 @@ __metadata: languageName: node linkType: hard -"folder-hash@npm:^4.0.2": +"folder-hash@npm:4.0.2": version: 4.0.2 resolution: "folder-hash@npm:4.0.2" dependencies: @@ -7113,29 +7103,6 @@ __metadata: languageName: unknown linkType: soft -"javascript-tests@workspace:tests/output/javascript": - version: 0.0.0-use.local - resolution: "javascript-tests@workspace:tests/output/javascript" - dependencies: - "@algolia/client-abtesting": 5.0.0 - "@algolia/client-analytics": 5.0.0 - "@algolia/client-common": 5.0.0 - "@algolia/client-insights": 5.0.0 - "@algolia/client-personalization": 5.0.0 - "@algolia/client-query-suggestions": 5.0.0 - "@algolia/client-search": 5.0.0 - "@algolia/client-sources": 0.0.1 - "@algolia/recommend": 5.0.0 - "@algolia/requester-node-http": 5.0.0 - "@types/jest": 27.0.3 - "@types/node": 16.11.11 - jest: 27.4.7 - ts-jest: 27.1.2 - ts-node: 10.5.0 - typescript: 4.5.4 - languageName: unknown - linkType: soft - "jest-changed-files@npm:^27.5.1": version: 27.5.1 resolution: "jest-changed-files@npm:27.5.1" @@ -10186,20 +10153,26 @@ __metadata: version: 0.0.0-use.local resolution: "scripts@workspace:scripts" dependencies: + "@apidevtools/swagger-parser": 10.0.3 "@octokit/rest": 18.12.0 - "@types/folder-hash": ^4.0.1 + "@types/folder-hash": 4.0.1 "@types/inquirer": 8.2.0 + "@types/jest": 27.4.0 "@types/js-yaml": 4.0.5 + "@types/mustache": 4.1.2 "@types/node": 16.11.11 - "@types/ora": ^3.2.0 + "@types/ora": 3.2.0 "@types/semver": 7.3.9 commander: 9.0.0 dotenv: 16.0.0 + eslint: 8.6.0 execa: 5.1.1 - folder-hash: ^4.0.2 + folder-hash: 4.0.2 inquirer: 8.2.0 jest: 27.4.7 js-yaml: 4.1.0 + mustache: 4.2.0 + openapi-types: 10.0.0 ora-classic: 5.4.2 semver: 7.3.5 ts-jest: 27.1.3 @@ -10790,23 +10763,6 @@ __metadata: languageName: node linkType: hard -"tests@workspace:tests": - version: 0.0.0-use.local - resolution: "tests@workspace:tests" - dependencies: - "@apidevtools/swagger-parser": 10.0.3 - "@types/jest": 27.4.0 - "@types/mustache": 4.1.2 - "@types/node": 16.11.11 - eslint: 8.6.0 - jest: 27.4.7 - mustache: 4.2.0 - openapi-types: 10.0.0 - ts-jest: 27.1.3 - typescript: 4.5.4 - languageName: unknown - linkType: soft - "text-table@npm:^0.2.0": version: 0.2.0 resolution: "text-table@npm:0.2.0" @@ -10913,40 +10869,6 @@ __metadata: languageName: node linkType: hard -"ts-jest@npm:27.1.2": - version: 27.1.2 - resolution: "ts-jest@npm:27.1.2" - dependencies: - bs-logger: 0.x - fast-json-stable-stringify: 2.x - jest-util: ^27.0.0 - json5: 2.x - lodash.memoize: 4.x - make-error: 1.x - semver: 7.x - yargs-parser: 20.x - peerDependencies: - "@babel/core": ">=7.0.0-beta.0 <8" - "@types/jest": ^27.0.0 - babel-jest: ">=27.0.0 <28" - esbuild: ~0.14.0 - jest: ^27.0.0 - typescript: ">=3.8 <5.0" - peerDependenciesMeta: - "@babel/core": - optional: true - "@types/jest": - optional: true - babel-jest: - optional: true - esbuild: - optional: true - bin: - ts-jest: cli.js - checksum: 2e7275f8a3545ec1340b37c458ace9244b5903e86861eb108beffff97d433f296c1254f76a41b573b1fe6245110b21bb62150bb88d55159f1dc7a929886535cb - languageName: node - linkType: hard - "ts-jest@npm:27.1.3": version: 27.1.3 resolution: "ts-jest@npm:27.1.3" From 3af376e6de3c88d33dc9657b1e14d1031bd8bd01 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Thu, 24 Feb 2022 14:48:13 +0100 Subject: [PATCH 04/20] update release --- scripts/release/create-release-issue.ts | 341 ++++++++++++------------ scripts/release/process-release.ts | 232 ++++++++-------- 2 files changed, 282 insertions(+), 291 deletions(-) diff --git a/scripts/release/create-release-issue.ts b/scripts/release/create-release-issue.ts index 9d99fbea4c..d425a2597f 100755 --- a/scripts/release/create-release-issue.ts +++ b/scripts/release/create-release-issue.ts @@ -3,9 +3,9 @@ import { Octokit } from '@octokit/rest'; import dotenv from 'dotenv'; import semver from 'semver'; -import openapitools from '../../openapitools.json'; +import { GENERATORS, LANGUAGES, run } from '../common'; -import { RELEASED_TAG, MAIN_BRANCH, OWNER, REPO, LANGS, run } from './common'; +import { RELEASED_TAG, MAIN_BRANCH, OWNER, REPO } from './common'; import TEXT from './text'; dotenv.config(); @@ -25,14 +25,11 @@ type Versions = { function readVersions(): Versions { const versions = {}; - const generators = openapitools['generator-cli'].generators; - - Object.keys(generators).forEach((generator) => { - const lang = generator.split('-')[0]; - if (!versions[lang]) { - versions[lang] = { - current: generators[generator].additionalProperties.packageVersion, - langName: lang, + Object.values(GENERATORS).forEach((gen) => { + if (!versions[gen.language]) { + versions[gen.language] = { + current: gen.additionalProperties.packageVersion, + langName: gen.language, next: undefined, }; } @@ -44,182 +41,188 @@ if (!process.env.GITHUB_TOKEN) { throw new Error('Environment variable `GITHUB_TOKEN` does not exist.'); } -if (run('git rev-parse --abbrev-ref HEAD') !== MAIN_BRANCH) { - throw new Error( - `You can run this script only from \`${MAIN_BRANCH}\` branch.` - ); -} +async function createReleaseIssue(): Promise { + if ((await run('git rev-parse --abbrev-ref HEAD')) !== MAIN_BRANCH) { + throw new Error( + `You can run this script only from \`${MAIN_BRANCH}\` branch.` + ); + } -if (run('git status --porcelain')) { - throw new Error( - 'Working directory is not clean. Commit all the changes first.' - ); -} + if (await run('git status --porcelain')) { + throw new Error( + 'Working directory is not clean. Commit all the changes first.' + ); + } -run(`git rev-parse --verify refs/tags/${RELEASED_TAG}`, { - errorMessage: '`released` tag is missing in this repository.', -}); + await run(`git rev-parse --verify refs/tags/${RELEASED_TAG}`, { + errorMessage: '`released` tag is missing in this repository.', + }); -// Reading versions from `openapitools.json` -const versions = readVersions(); + // Reading versions from `openapitools.json` + const versions = readVersions(); + + console.log('Pulling from origin...'); + run(`git pull origin ${MAIN_BRANCH}`); + + console.log('Pushing to origin...'); + run(`git push origin ${MAIN_BRANCH}`); + + const commitsWithoutScope: string[] = []; + const commitsWithNonLanguageScope: string[] = []; + + // Reading commits since last release + type LatestCommit = { + hash: string; + type: string; + lang: string; + message: string; + raw: string; + }; + const latestCommits = ( + await run(`git log --oneline ${RELEASED_TAG}..${MAIN_BRANCH}`) + ) + .split('\n') + .filter(Boolean) + .map((commit) => { + const hash = commit.slice(0, 7); + let message = commit.slice(8); + let type = message.slice(0, message.indexOf(':')); + const matchResult = type.match(/(.+)\((.+)\)/); + if (!matchResult) { + commitsWithoutScope.push(commit); + return undefined; + } + message = message.slice(message.indexOf(':') + 1).trim(); + type = matchResult[1]; + const lang = matchResult[2]; + + if (!LANGUAGES.includes(lang)) { + commitsWithNonLanguageScope.push(commit); + return undefined; + } + + return { + hash, + type, // `fix` | `feat` | `chore` | ... + lang, // `javascript` | `php` | `java` | ... + message, + raw: commit, + }; + }) + .filter(Boolean) as LatestCommit[]; -console.log('Pulling from origin...'); -run(`git pull origin ${MAIN_BRANCH}`); + console.log('[INFO] Skipping these commits due to lack of language scope:'); + console.log(commitsWithoutScope.map((commit) => ` ${commit}`).join('\n')); -console.log('Pushing to origin...'); -run(`git push origin ${MAIN_BRANCH}`); + console.log(''); + console.log('[INFO] Skipping these commits due to wrong scopes:'); + console.log( + commitsWithNonLanguageScope.map((commit) => ` ${commit}`).join('\n') + ); -const commitsWithoutScope: string[] = []; -const commitsWithNonLanguageScope: string[] = []; + LANGUAGES.forEach((lang) => { + const commits = latestCommits.filter( + (lastestCommit) => lastestCommit.lang === lang + ); + const currentVersion = versions[lang].current; -// Reading commits since last release -type LatestCommit = { - hash: string; - type: string; - lang: string; - message: string; - raw: string; -}; -const latestCommits = run(`git log --oneline ${RELEASED_TAG}..${MAIN_BRANCH}`) - .split('\n') - .filter(Boolean) - .map((commit) => { - const hash = commit.slice(0, 7); - let message = commit.slice(8); - let type = message.slice(0, message.indexOf(':')); - const matchResult = type.match(/(.+)\((.+)\)/); - if (!matchResult) { - commitsWithoutScope.push(commit); - return undefined; + if (commits.length === 0) { + versions[lang].next = currentVersion; + versions[lang].noCommit = true; + return; } - message = message.slice(message.indexOf(':') + 1).trim(); - type = matchResult[1]; - const lang = matchResult[2]; - if (!LANGS.includes(lang)) { - commitsWithNonLanguageScope.push(commit); - return undefined; + if (semver.prerelease(currentVersion)) { + // if version is like 0.1.2-beta.1, it increases to 0.1.2-beta.2, even if there's a breaking change. + versions[lang].next = semver.inc(currentVersion, 'prerelease'); + return; } - return { - hash, - type, // `fix` | `feat` | `chore` | ... - lang, // `javascript` | `php` | `java` | ... - message, - raw: commit, - }; - }) - .filter(Boolean) as LatestCommit[]; - -console.log('[INFO] Skipping these commits due to lack of language scope:'); -console.log(commitsWithoutScope.map((commit) => ` ${commit}`).join('\n')); - -console.log(''); -console.log('[INFO] Skipping these commits due to wrong scopes:'); -console.log( - commitsWithNonLanguageScope.map((commit) => ` ${commit}`).join('\n') -); - -LANGS.forEach((lang) => { - const commits = latestCommits.filter( - (lastestCommit) => lastestCommit.lang === lang - ); - const currentVersion = versions[lang].current; - - if (commits.length === 0) { - versions[lang].next = currentVersion; - versions[lang].noCommit = true; - return; - } - - if (semver.prerelease(currentVersion)) { - // if version is like 0.1.2-beta.1, it increases to 0.1.2-beta.2, even if there's a breaking change. - versions[lang].next = semver.inc(currentVersion, 'prerelease'); - return; - } - - if (commits.some((commit) => commit.message.includes('BREAKING CHANGE'))) { - versions[lang].next = semver.inc(currentVersion, 'major'); - return; - } - - const commitTypes = new Set(commits.map(({ type }) => type)); - if (commitTypes.has('feat')) { - versions[lang].next = semver.inc(currentVersion, 'minor'); - return; - } + if (commits.some((commit) => commit.message.includes('BREAKING CHANGE'))) { + versions[lang].next = semver.inc(currentVersion, 'major'); + return; + } - versions[lang].next = semver.inc(currentVersion, 'patch'); - if (!commitTypes.has('fix')) { - versions[lang].skipRelease = true; - } -}); + const commitTypes = new Set(commits.map(({ type }) => type)); + if (commitTypes.has('feat')) { + versions[lang].next = semver.inc(currentVersion, 'minor'); + return; + } -const versionChanges = LANGS.map((lang) => { - const { current, next, noCommit, skipRelease, langName } = versions[lang]; + versions[lang].next = semver.inc(currentVersion, 'patch'); + if (!commitTypes.has('fix')) { + versions[lang].skipRelease = true; + } + }); - if (noCommit) { - return `- ~${langName}: v${current} (${TEXT.noCommit})~`; - } + const versionChanges = LANGUAGES.map((lang) => { + const { current, next, noCommit, skipRelease, langName } = versions[lang]; - if (!current) { - return `- ~${langName}: (${TEXT.currentVersionNotFound})~`; - } + if (noCommit) { + return `- ~${langName}: v${current} (${TEXT.noCommit})~`; + } - const checked = skipRelease ? ' ' : 'x'; - return [ - `- [${checked}] ${langName}: v${current} -> v${next}`, - skipRelease && TEXT.descriptionForSkippedLang(langName), - ] - .filter(Boolean) - .join('\n'); -}).join('\n'); - -const changelogs = LANGS.filter( - (lang) => !versions[lang].noCommit && versions[lang].current -) - .flatMap((lang) => { - if (versions[lang].noCommit) { - return []; + if (!current) { + return `- ~${langName}: (${TEXT.currentVersionNotFound})~`; } + const checked = skipRelease ? ' ' : 'x'; return [ - `### ${versions[lang].langName}`, - ...latestCommits - .filter((commit) => commit.lang === lang) - .map((commit) => `- ${commit.raw}`), - ]; - }) - .join('\n'); - -const body = [ - TEXT.header, - TEXT.versionChangeHeader, - versionChanges, - TEXT.changelogHeader, - TEXT.changelogDescription, - changelogs, - TEXT.approvalHeader, - TEXT.approval, -].join('\n\n'); - -const octokit = new Octokit({ - auth: `token ${process.env.GITHUB_TOKEN}`, -}); - -octokit.rest.issues - .create({ - owner: OWNER, - repo: REPO, - title: `chore: release ${new Date().toISOString().split('T')[0]}`, - body, - }) - .then((result) => { - const { - data: { number, html_url: url }, - } = result; - - console.log(''); - console.log(`Release issue #${number} is ready for review.`); - console.log(` > ${url}`); + `- [${checked}] ${langName}: v${current} -> v${next}`, + skipRelease && TEXT.descriptionForSkippedLang(langName), + ] + .filter(Boolean) + .join('\n'); + }).join('\n'); + + const changelogs = LANGUAGES.filter( + (lang) => !versions[lang].noCommit && versions[lang].current + ) + .flatMap((lang) => { + if (versions[lang].noCommit) { + return []; + } + + return [ + `### ${versions[lang].langName}`, + ...latestCommits + .filter((commit) => commit.lang === lang) + .map((commit) => `- ${commit.raw}`), + ]; + }) + .join('\n'); + + const body = [ + TEXT.header, + TEXT.versionChangeHeader, + versionChanges, + TEXT.changelogHeader, + TEXT.changelogDescription, + changelogs, + TEXT.approvalHeader, + TEXT.approval, + ].join('\n\n'); + + const octokit = new Octokit({ + auth: `token ${process.env.GITHUB_TOKEN}`, }); + + octokit.rest.issues + .create({ + owner: OWNER, + repo: REPO, + title: `chore: release ${new Date().toISOString().split('T')[0]}`, + body, + }) + .then((result) => { + const { + data: { number, html_url: url }, + } = result; + + console.log(''); + console.log(`Release issue #${number} is ready for review.`); + console.log(` > ${url}`); + }); +} + +createReleaseIssue(); diff --git a/scripts/release/process-release.ts b/scripts/release/process-release.ts index d8bb376be6..80111af269 100755 --- a/scripts/release/process-release.ts +++ b/scripts/release/process-release.ts @@ -1,30 +1,17 @@ /* eslint-disable no-console */ import fs from 'fs'; -import path from 'path'; import dotenv from 'dotenv'; import execa from 'execa'; import openapitools from '../../openapitools.json'; +import { getAbsolutePath, run } from '../common'; -import type { Run } from './common'; -import { - MAIN_BRANCH, - OWNER, - REPO, - run as runOriginal, - getMarkdownSection, -} from './common'; +import { MAIN_BRANCH, OWNER, REPO, getMarkdownSection } from './common'; import TEXT from './text'; -// This script is run by `yarn workspace ...`, which means the current working directory is `./script` -const ROOT_DIR = path.resolve(process.cwd(), '..'); - dotenv.config(); -const run: Run = (command, options = {}) => - runOriginal(command, { cwd: ROOT_DIR, ...options }); - if (!process.env.GITHUB_TOKEN) { throw new Error('Environment variable `GITHUB_TOKEN` does not exist.'); } @@ -33,121 +20,122 @@ if (!process.env.EVENT_NUMBER) { throw new Error('Environment variable `EVENT_NUMBER` does not exist.'); } -const issueBody = JSON.parse( - execa.sync('curl', [ - '-H', - `Authorization: token ${process.env.GITHUB_TOKEN}`, - `https://api.github.com/repos/${OWNER}/${REPO}/issues/${process.env.EVENT_NUMBER}`, - ]).stdout -).body; +async function processRelease(): Promise { + const issueBody = JSON.parse( + execa.sync('curl', [ + '-H', + `Authorization: token ${process.env.GITHUB_TOKEN}`, + `https://api.github.com/repos/${OWNER}/${REPO}/issues/${process.env.EVENT_NUMBER}`, + ]).stdout + ).body; + + if ( + !getMarkdownSection(issueBody, TEXT.approvalHeader) + .split('\n') + .find((line) => line.startsWith(`- [x] ${TEXT.approved}`)) + ) { + throw new Error('The issue was not approved.'); + } -if ( - !getMarkdownSection(issueBody, TEXT.approvalHeader) + const versionsToRelease = {}; + getMarkdownSection(issueBody, TEXT.versionChangeHeader) .split('\n') - .find((line) => line.startsWith(`- [x] ${TEXT.approved}`)) -) { - throw new Error('The issue was not approved.'); -} - -const versionsToRelease = {}; -getMarkdownSection(issueBody, TEXT.versionChangeHeader) - .split('\n') - .forEach((line) => { - const result = line.match(/- \[x\] (.+): v(.+) -> v(.+)/); - if (!result) { - return; + .forEach((line) => { + const result = line.match(/- \[x\] (.+): v(.+) -> v(.+)/); + if (!result) { + return; + } + const [, lang, current, next] = result; + versionsToRelease[lang] = { + current, + next, + }; + }); + + const langsToUpdateRepo = getMarkdownSection( + issueBody, + TEXT.versionChangeHeader + ) + .split('\n') + .map((line) => { + const result = line.match(/- \[ \] (.+): v(.+) -> v(.+)/); + return result?.[1]; + }) + .filter(Boolean); // e.g. ['javascript', 'php'] + + // update versions in `openapitools.json` + Object.keys(openapitools['generator-cli'].generators).forEach((client) => { + const lang = client.split('-')[0]; + if (versionsToRelease[lang]) { + openapitools['generator-cli'].generators[ + client + ].additionalProperties.packageVersion = versionsToRelease[lang].next; } - const [, lang, current, next] = result; - versionsToRelease[lang] = { - current, - next, - }; }); + fs.writeFileSync( + getAbsolutePath('openapitools.json'), + JSON.stringify(openapitools, null, 2) + ); + + // update changelogs + new Set([...Object.keys(versionsToRelease), ...langsToUpdateRepo]).forEach( + (lang) => { + const filePath = getAbsolutePath(`docs/changelogs/${lang}.md`); + const header = versionsToRelease[lang!] + ? `## ${versionsToRelease[lang!].next}` + : `## ${new Date().toISOString().split('T')[0]}`; + const newChangelog = getMarkdownSection( + getMarkdownSection(issueBody, TEXT.changelogHeader), + `### ${lang}` + ); + const existingContent = fs.readFileSync(filePath).toString(); + fs.writeFileSync( + filePath, + [header, newChangelog, existingContent].join('\n\n') + ); + } + ); + + // commit openapitools and changelogs + if (process.env.RELEASE_TEST !== 'true') { + await run('git config user.name "api-clients-bot"'); + await run('git config user.email "bot@algolia.com"'); + await run('git add openapitools.json'); + await run('git add doc/changelogs/*'); + execa.sync('git', ['commit', '-m', TEXT.commitMessage]); + await run(`git push origin ${MAIN_BRANCH}`); + } -const langsToUpdateRepo = getMarkdownSection( - issueBody, - TEXT.versionChangeHeader -) - .split('\n') - .map((line) => { - const result = line.match(/- \[ \] (.+): v(.+) -> v(.+)/); - return result?.[1]; - }) - .filter(Boolean); // e.g. ['javascript', 'php'] - -// update versions in `openapitools.json` -Object.keys(openapitools['generator-cli'].generators).forEach((client) => { - const lang = client.split('-')[0]; - if (versionsToRelease[lang]) { - openapitools['generator-cli'].generators[ - client - ].additionalProperties.packageVersion = versionsToRelease[lang].next; + // generate clients to release + for (const lang of Object.keys(versionsToRelease)) { + console.log(`Generating ${lang} client(s)...`); + await run(`yarn api generate ${lang} all`); } -}); -fs.writeFileSync( - path.resolve(ROOT_DIR, 'openapitools.json'), - JSON.stringify(openapitools, null, 2) -); - -// update changelogs -new Set([...Object.keys(versionsToRelease), ...langsToUpdateRepo]).forEach( - (lang) => { - const filePath = path.resolve(ROOT_DIR, `docs/changelogs/${lang}.md`); - const header = versionsToRelease[lang!] - ? `## ${versionsToRelease[lang!].next}` - : `## ${new Date().toISOString().split('T')[0]}`; - const newChangelog = getMarkdownSection( - getMarkdownSection(issueBody, TEXT.changelogHeader), - `### ${lang}` - ); - const existingContent = fs.readFileSync(filePath).toString(); - fs.writeFileSync( - filePath, - [header, newChangelog, existingContent].join('\n\n') - ); + + // generate clients to just update the repos + for (const lang of langsToUpdateRepo) { + console.log(`Generating ${lang} client(s)...`); + await run(`yarn api generate ${lang} all`, { verbose: true }); } -); - -// commit openapitools and changelogs -if (process.env.RELEASE_TEST !== 'true') { - run('git config user.name "api-clients-bot"'); - run('git config user.email "bot@algolia.com"'); - run('git add openapitools.json'); - run('git add doc/changelogs/*'); - execa.sync('git', ['commit', '-m', TEXT.commitMessage]); - run(`git push origin ${MAIN_BRANCH}`); -} -// generate clients to release -Object.keys(versionsToRelease).forEach((lang) => { - console.log(`Generating ${lang} client(s)...`); - // @ts-expect-error the library `execa` is not typed correctly - run(`yarn generate ${lang}`).pipe(process.stdout); -}); - -// generate clients to just update the repos -langsToUpdateRepo.forEach((lang) => { - console.log(`Generating ${lang} client(s)...`); - // @ts-expect-error the library `execa` is not typed correctly - run(`yarn generate ${lang}`).pipe(process.stdout); -}); - -const clientPath = path.resolve( - ROOT_DIR, - 'clients/dummy-algoliasearch-client-javascript' -); -const runInClient: Run = (command, options = {}) => - runOriginal(command, { + const clientPath = getAbsolutePath( + 'clients/dummy-algoliasearch-client-javascript' + ); + const runInClient = (command: string, options = {}): Promise => + run(command, { + cwd: clientPath, + ...options, + }); + + await runInClient(`git checkout next`); + await run( + `cp -r clients/algoliasearch-client-javascript/ clients/dummy-algoliasearch-client-javascript` + ); + await runInClient(`git add .`); + execa.sync('git', ['commit', '-m', 'chore: release test'], { cwd: clientPath, - ...options, }); + await runInClient(`git push origin next`); +} -runInClient(`git checkout next`); -run( - `cp -r clients/algoliasearch-client-javascript/ clients/dummy-algoliasearch-client-javascript` -); -runInClient(`git add .`); -execa.sync('git', ['commit', '-m', 'chore: release test'], { - cwd: clientPath, -}); -runInClient(`git push origin next`); +processRelease(); From bfc7c4a3f152a0e0de2bae317faf398e6824f758 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Thu, 24 Feb 2022 15:07:55 +0100 Subject: [PATCH 05/20] tiny bit faster --- base.tsconfig.json | 3 ++- package.json | 2 +- scripts/tsconfig.json | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/base.tsconfig.json b/base.tsconfig.json index e57897d3dc..4d0d503a84 100644 --- a/base.tsconfig.json +++ b/base.tsconfig.json @@ -17,5 +17,6 @@ "typeRoots": ["node_modules/@types"], "types": ["node"], "resolveJsonModule": true - } + }, + "exclude": ["node_modules"] } diff --git a/package.json b/package.json index 80fdc23814..cf32914970 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "scripts/" ], "scripts": { - "api": "yarn workspace scripts ts-node ./index.ts", + "api": "yarn workspace scripts ts-node --transpile-only ./index.ts", "clean": "rm -rf **/dist **/build **/node_modules **/.gradle", "cts:test:scripts": "yarn workspace scripts test", "cts:lint:scripts": "eslint --ext=ts scripts/", diff --git a/scripts/tsconfig.json b/scripts/tsconfig.json index 34cc31a9fd..ab90a217cc 100644 --- a/scripts/tsconfig.json +++ b/scripts/tsconfig.json @@ -5,6 +5,6 @@ "types": ["node", "jest"], "outDir": "dist" }, - "include": ["pre-gen/setHostsOptions.ts", "release/*"], - "exclude": ["dist", "*.json"] + "include": ["**/*.ts"], + "exclude": ["dist", "*.json", "node_modules"] } From 6bcf0178ee234d6d1a81f29a6728516b253cc0e1 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Thu, 24 Feb 2022 15:34:49 +0100 Subject: [PATCH 06/20] clean up --- package.json | 9 +++--- scripts/generate.ts | 22 +++++++++++++++ scripts/index.ts | 16 +++++------ scripts/post-gen/global.sh | 58 -------------------------------------- 4 files changed, 33 insertions(+), 72 deletions(-) delete mode 100755 scripts/post-gen/global.sh diff --git a/package.json b/package.json index cf32914970..b57ea1185d 100644 --- a/package.json +++ b/package.json @@ -5,24 +5,23 @@ "clients/algoliasearch-client-javascript/", "playground/javascript/node/", "playground/javascript/browser/", - "tests/", "scripts/" ], "scripts": { "api": "yarn workspace scripts ts-node --transpile-only ./index.ts", "clean": "rm -rf **/dist **/build **/node_modules **/.gradle", - "cts:test:scripts": "yarn workspace scripts test", - "cts:lint:scripts": "eslint --ext=ts scripts/", "docker:build": "./scripts/docker/build.sh", "docker:clean": "docker stop dev; docker rm -f dev; docker image rm -f api-clients-automation", "docker:mount": "./scripts/docker/mount.sh", "docker:setup": "yarn docker:clean && yarn docker:build && yarn docker:mount", "docker": "docker exec -it dev yarn api $*", + "github-actions:lint": "eslint --ext=yml .github/", + "lint:scripts": "eslint --ext=ts scripts/", "playground:browser": "yarn workspace javascript-browser-playground start", "specs:fix": "eslint --ext=yml specs/ --fix", + "release": "yarn workspace scripts createReleaseIssue", "specs:lint": "eslint --ext=yml specs/$0", - "github-actions:lint": "eslint --ext=yml .github/", - "release": "yarn workspace scripts createReleaseIssue" + "test:scripts": "yarn workspace scripts test" }, "devDependencies": { "@openapitools/openapi-generator-cli": "2.4.26", diff --git a/scripts/generate.ts b/scripts/generate.ts index 3ca1bf890d..653ce9bcac 100644 --- a/scripts/generate.ts +++ b/scripts/generate.ts @@ -62,5 +62,27 @@ export async function generate( const langs = [...new Set(generators.map((gen) => gen.language))]; for (const lang of langs) { await formatter(lang, getLanguageFolder(lang), verbose); + + if (lang === 'javascript') { + const spinner = createSpinner( + 'Cleaning JavaScript client utils', + verbose + ); + await run('yarn workspace algoliasearch-client-javascript clean:utils', { + verbose, + }); + spinner.text = 'Building JavaScript client utils'; + await run('yarn workspace algoliasearch-client-javascript build:utils', { + verbose, + }); + + spinner.succeed(); + } + } + + if (!CI) { + const spinner = createSpinner('formatting specs', verbose).start(); + await run(`yarn specs:fix`, { verbose }); + spinner.succeed(); } } diff --git a/scripts/index.ts b/scripts/index.ts index 7b8bd7ccaa..c8e4cc9779 100644 --- a/scripts/index.ts +++ b/scripts/index.ts @@ -32,6 +32,9 @@ async function promptLanguage(defaut: string | undefined): Promise { if (defaut) { return defaut; } + if (CI) { + return 'all'; + } const { language } = await inquirer.prompt([ { type: 'list', @@ -51,6 +54,9 @@ async function promptClient( if (defaut) { return defaut; } + if (CI) { + return 'all'; + } const { client } = await inquirer.prompt([ { type: 'list', @@ -165,15 +171,7 @@ buildCommand client = await promptClient(client); if (!outputFormat) { - ({ outputFormat } = await inquirer.prompt([ - { - type: 'list', - name: 'outputFormat', - message: 'Select the output format', - default: 'yml', - choices: ['yml', 'json'], - }, - ])); + outputFormat = 'yml'; } let clientsTodo = [client]; diff --git a/scripts/post-gen/global.sh b/scripts/post-gen/global.sh deleted file mode 100755 index 213cd0a169..0000000000 --- a/scripts/post-gen/global.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -# Break on non-zero code -set -e - -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" -# Move to the root (easier to locate other scripts) -cd ${DIR}/../.. - -LANGUAGE=$1 - -if [[ $CI ]]; then - echo "Not running post-gen/global on CI for $LANGUAGE" - - exit 0 -fi - -if [[ ! $DOCKER ]]; then - echo "You should run scripts via the docker container, see README.md" - - exit 1 -fi - -build_js_utils() { - echo "> Cleaning JavaScript client utils..." - yarn workspace algoliasearch-client-javascript clean:utils - - echo "> Building JavaScript client utils..." - yarn workspace algoliasearch-client-javascript build:utils - - echo "" -} - -format_specs() { - echo "> Formatting specs..." - - CMD="yarn specs:fix" - if [[ $VERBOSE == "true" ]]; then - $CMD - else - set +e - log=$($CMD) - - if [[ $? != 0 ]]; then - echo "$log" - exit 1 - fi - set -e - fi - - echo "" -} - -format_specs - -if [[ $LANGUAGE == 'javascript' || $LANGUAGE == 'all' ]]; then - build_js_utils -fi From 0a0f3162de44be756c24624ae54043c1ecf248d4 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Thu, 24 Feb 2022 15:34:59 +0100 Subject: [PATCH 07/20] doc --- .github/workflows/check.yml | 29 +++++++++++++---------------- README.md | 22 +++++++++++++--------- docs/CTS.md | 14 +++++++------- 3 files changed, 33 insertions(+), 32 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index d852f93364..8520105168 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -60,7 +60,7 @@ jobs: uses: ./.github/actions/cache - name: Building ${{ matrix.client }} specs - run: yarn build:specs ${{ matrix.client }} + run: yarn api build specs ${{ matrix.client }} - name: Check diff with pushed spec run: exit $(git status --porcelain specs/bundled/${{ matrix.client }}.yml | wc -l) @@ -123,7 +123,7 @@ jobs: - name: Generate ${{ matrix.client.name }} client if: steps.cache.outputs.cache-hit != 'true' - run: yarn generate javascript ${{ matrix.client.name }} + run: yarn api generate javascript ${{ matrix.client.name }} - name: Check diff with pushed client if: steps.cache.outputs.cache-hit != 'true' @@ -131,7 +131,7 @@ jobs: - name: Build ${{ matrix.client.name }} client if: steps.cache.outputs.cache-hit != 'true' - run: yarn build:clients javascript ${{ matrix.client.name }} + run: yarn api build clients javascript ${{ matrix.client.name }} client_java: runs-on: ubuntu-20.04 @@ -160,7 +160,7 @@ jobs: - name: Generate ${{ matrix.client.name }} client if: steps.cache.outputs.cache-hit != 'true' - run: yarn generate java ${{ matrix.client.name }} + run: yarn api generate java ${{ matrix.client.name }} - name: Check diff with pushed client if: steps.cache.outputs.cache-hit != 'true' @@ -168,7 +168,7 @@ jobs: - name: Build ${{ matrix.client.name }} client if: steps.cache.outputs.cache-hit != 'true' - run: yarn build:clients java ${{ matrix.client.name }} + run: yarn api build clients java ${{ matrix.client.name }} client_php: runs-on: ubuntu-20.04 @@ -190,7 +190,7 @@ jobs: - name: Generate ${{ matrix.client.name }} client if: steps.cache.outputs.cache-hit != 'true' - run: yarn generate php ${{ matrix.client.name }} + run: yarn api generate php ${{ matrix.client.name }} - name: Check diff with pushed client if: steps.cache.outputs.cache-hit != 'true' @@ -198,7 +198,7 @@ jobs: - name: Build ${{ matrix.client.name }} client if: steps.cache.outputs.cache-hit != 'true' - run: yarn build:clients php ${{ matrix.client.name }} + run: yarn api build clients php ${{ matrix.client.name }} cts: runs-on: ubuntu-20.04 @@ -217,20 +217,14 @@ jobs: with: job: cts - - name: Check script linting - run: yarn cts:lint:scripts - - - name: Test CTS script - run: yarn cts:test:scripts - - name: Generate CTS - run: yarn cts:generate + run: yarn api cts generate - name: Check diff with pushed CTS run: exit $(git status --porcelain ./tests/output | wc -l) - name: Run CTS - run: yarn cts:test + run: yarn api cts run scripts: runs-on: ubuntu-20.04 @@ -243,5 +237,8 @@ jobs: id: restore uses: ./.github/actions/cache + - name: Check script linting + run: yarn lint:scripts + - name: Test scripts - run: yarn workspace scripts test + run: yarn test:scripts diff --git a/README.md b/README.md index 344afbbb9c..a989f587d5 100644 --- a/README.md +++ b/README.md @@ -53,19 +53,19 @@ You can make changes locally and run commands through the docker container. #### Usage ```bash -yarn docker build:specs +yarn docker build specs ``` #### Build all specs ```bash -yarn docker build:specs +yarn docker build specs all ``` #### Build specific spec ```bash -yarn docker build:specs recommend +yarn docker build specs recommend ``` #### Fix the specs format @@ -73,14 +73,14 @@ yarn docker build:specs recommend This is used by the build script and should not need to be called manually but if you want to format all specs file do: ```bash -yarn docker specs:fix +yarn specs:fix ``` If you just want to check the format (not override the files), run: ```bash -yarn docker specs:lint -yarn docker specs:lint search +yarn specs:lint +yarn specs:lint search ``` ### Generate clients based on the [`specs`](./specs/) @@ -94,7 +94,7 @@ yarn docker generate #### Generate all clients ```bash -yarn docker generate +yarn docker generate all all ``` ### Generate specific client for specific language @@ -102,15 +102,19 @@ yarn docker generate #### Usage ```bash -yarn docker build:clients +yarn docker build clients ``` ### Build specific client for specific language ```bash -yarn docker build:clients java recommend +yarn docker build clients java recommend ``` +### Verbose command + +You can add `-v` to almost every command to have a more verbose output. + ## Testing clients You can test our generated clients by running: diff --git a/docs/CTS.md b/docs/CTS.md index 2d2c5badbf..3b3023a427 100644 --- a/docs/CTS.md +++ b/docs/CTS.md @@ -8,28 +8,28 @@ It is automaticaly generated for all languages, from a JSON entry point. > CTS requires all clients to be built ```bash -yarn docker build:specs -yarn docker build:clients -yarn docker cts:generate -yarn docker cts:test +yarn docker build specs all +yarn docker build clients all all +yarn docker cts generate all all +yarn docker cts run all ``` If you only want to generate the tests for a language, you can run: ```bash -yarn docker cts:generate javascript +yarn docker cts generate javascript all ``` Or for a specific client: ```bash -yarn docker cts:generate all search +yarn docker cts generate all search ``` Or a specific language and client: ```bash -yarn docker cts:generate javascript search +yarn docker cts generate javascript search ``` ## How to add test From a6228a342c9766dc75c1ffd7ed42049af5db8927 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Thu, 24 Feb 2022 16:10:04 +0100 Subject: [PATCH 08/20] add php --- clients.config.json | 3 +++ openapitools.json | 2 +- scripts/buildSpecs.ts | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/clients.config.json b/clients.config.json index a8bf220c37..1b5a0c15bf 100644 --- a/clients.config.json +++ b/clients.config.json @@ -12,5 +12,8 @@ "extension": ".test.java", "outputFolder": "src/test/java/com/algolia" } + }, + "php": { + "folder": "clients/algoliasearch-client-php" } } diff --git a/openapitools.json b/openapitools.json index 491b4a390a..2e82b8a813 100644 --- a/openapitools.json +++ b/openapitools.json @@ -396,4 +396,4 @@ } } } -} +} \ No newline at end of file diff --git a/scripts/buildSpecs.ts b/scripts/buildSpecs.ts index bc9370da06..18994e3cc2 100644 --- a/scripts/buildSpecs.ts +++ b/scripts/buildSpecs.ts @@ -25,7 +25,7 @@ async function buildSpec( const storedHash = (await fsp.readFile(cacheFile)).toString(); if (storedHash === hash) { spinner.succeed( - 'skipped building spec because the files did not change' + `skipped building ${client} spec because the files did not change` ); return; } From 10f1a01d8f53801942fb68783a316152bde769b5 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Thu, 24 Feb 2022 16:10:16 +0100 Subject: [PATCH 09/20] optimize js on ci --- scripts/generate.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/scripts/generate.ts b/scripts/generate.ts index 653ce9bcac..e68c982f68 100644 --- a/scripts/generate.ts +++ b/scripts/generate.ts @@ -50,24 +50,30 @@ export async function generate( const spinner = createSpinner(`pre-gen ${gen.key}`, verbose).start(); await preGen(gen, verbose); - spinner.text = `generation ${gen.key}`; + spinner.text = `generating ${gen.key}`; await generateClient(gen, verbose); spinner.text = `post-gen ${gen.key}`; await postGen(gen, verbose); + if (gen.language === 'javascript' && CI) { + // because the CI is parallelized, run the formatter for each client + await formatter(gen.language, gen.output, verbose); + } + spinner.succeed(); } const langs = [...new Set(generators.map((gen) => gen.language))]; for (const lang of langs) { - await formatter(lang, getLanguageFolder(lang), verbose); - + if (!CI || lang !== 'javascript') { + await formatter(lang, getLanguageFolder(lang), verbose); + } if (lang === 'javascript') { const spinner = createSpinner( 'Cleaning JavaScript client utils', verbose - ); + ).start(); await run('yarn workspace algoliasearch-client-javascript clean:utils', { verbose, }); From e22626a98623e5ab8a0f66a8ec66f0b940015399 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Thu, 24 Feb 2022 16:30:29 +0100 Subject: [PATCH 10/20] fix build --- scripts/buildClients.ts | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/scripts/buildClients.ts b/scripts/buildClients.ts index 4faf0bc5ff..09e83d65f9 100644 --- a/scripts/buildClients.ts +++ b/scripts/buildClients.ts @@ -1,12 +1,13 @@ import { run } from './common'; +import { getLanguageFolder } from './config'; import { createSpinner } from './oraLog'; import type { Generator } from './types'; -async function buildClient( - { language, additionalProperties: { packageName } }: Generator, +async function buildPerClient( + { language, key, additionalProperties: { packageName } }: Generator, verbose: boolean ): Promise { - const spinner = createSpinner(`building ${language}`, verbose).start(); + const spinner = createSpinner(`building ${key}`, verbose).start(); switch (language) { case 'javascript': await run(`yarn workspace ${packageName} clean`, { verbose }); @@ -15,9 +16,22 @@ async function buildClient( { verbose } ); break; + default: + } + spinner.succeed(); +} + +async function buildAllClients( + language: string, + verbose: boolean +): Promise { + const spinner = createSpinner(`building ${language}`, verbose).start(); + switch (language) { case 'java': await run( - `./gradle/gradlew --no-daemon -p clients/${packageName} assemble`, + `./gradle/gradlew --no-daemon -p ${getLanguageFolder( + language + )} assemble`, { verbose, } @@ -35,6 +49,17 @@ export async function buildClients( verbose: boolean ): Promise { for (const gen of generators) { - await buildClient(gen, verbose); + // we should detect this from the config file, for example `isMutliBuild` + if (gen.language === 'javascript') { + await buildPerClient(gen, verbose); + } + } + + // for other languages, we can mutualize the build + const langs = [...new Set(generators.map((gen) => gen.language))]; + for (const lang of langs) { + if (lang !== 'javascript') { + await buildAllClients(lang, verbose); + } } } From 1dc8894e69575032438508a662e1a71439b98e19 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Thu, 24 Feb 2022 17:49:40 +0100 Subject: [PATCH 11/20] swap config --- clients.config.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/clients.config.json b/clients.config.json index 1b5a0c15bf..817a6a11af 100644 --- a/clients.config.json +++ b/clients.config.json @@ -2,15 +2,15 @@ "java": { "folder": "clients/algoliasearch-client-java-2", "tests": { - "extension": ".test.ts", - "outputFolder": "src" + "extension": ".test.java", + "outputFolder": "src/test/java/com/algolia" } }, "javascript": { "folder": "clients/algoliasearch-client-javascript", - "texts": { - "extension": ".test.java", - "outputFolder": "src/test/java/com/algolia" + "tests": { + "extension": ".test.ts", + "outputFolder": "src" } }, "php": { From c4e075963528c2e154197568e6dbc22efa37af9b Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Thu, 24 Feb 2022 17:49:51 +0100 Subject: [PATCH 12/20] log improvement --- scripts/common.ts | 4 ++-- scripts/cts/client/generate.ts | 18 ++++++++++----- scripts/cts/generate.ts | 7 +++++- scripts/cts/methods/requests/cts.ts | 27 +++++++--------------- scripts/cts/methods/requests/generate.ts | 17 ++++++++++---- scripts/cts/utils.ts | 10 ++++---- scripts/formatter.ts | 5 +++- scripts/oraLog.ts | 29 ++++++++++++++++++------ scripts/release/process-release.ts | 8 +++---- 9 files changed, 75 insertions(+), 50 deletions(-) diff --git a/scripts/common.ts b/scripts/common.ts index 7cbe05e805..58fb5a3c95 100644 --- a/scripts/common.ts +++ b/scripts/common.ts @@ -75,7 +75,7 @@ export async function exists(ppath: string): Promise { } } -export function getAbsolutePath(ppath: string): string { +export function toAbsolutePath(ppath: string): string { return path.resolve(ROOT_DIR, ppath); } @@ -84,7 +84,7 @@ export async function runIfExists( args: string, opts: RunOptions = {} ): Promise { - if (await exists(getAbsolutePath(scriptFile))) { + if (await exists(toAbsolutePath(scriptFile))) { return await run(`${scriptFile} ${args}`, opts); } return ''; diff --git a/scripts/cts/client/generate.ts b/scripts/cts/client/generate.ts index f4cc76d1c3..20a74cbd95 100644 --- a/scripts/cts/client/generate.ts +++ b/scripts/cts/client/generate.ts @@ -2,7 +2,7 @@ import fsp from 'fs/promises'; import Mustache from 'mustache'; -import { exists, getAbsolutePath } from '../../common'; +import { exists, toAbsolutePath } from '../../common'; import { createSpinner } from '../../oraLog'; import type { Generator } from '../../types'; import { @@ -19,7 +19,7 @@ const testPath = 'client'; async function loadTests(client: string): Promise { const testsBlocks: TestsBlock[] = []; - const clientPath = getAbsolutePath(`tests/CTS/client/${client}`); + const clientPath = toAbsolutePath(`tests/CTS/client/${client}`); if (!(await exists(clientPath))) { return []; @@ -65,15 +65,21 @@ export async function generateClientTests( }: Generator, verbose: boolean ): Promise { - let spinner = createSpinner('generating client tests', verbose).start(); + let spinner = createSpinner( + { text: 'generating client tests', indent: 4 }, + verbose + ).start(); const testsBlocks = await loadTests(client); if (testsBlocks.length === 0) { - spinner.warn("Skipping because tests doesn' exist"); + spinner.warn("skipping because tests doesn't exist"); return; } spinner.info(); - spinner = createSpinner('loading templates', verbose).start(); + spinner = createSpinner( + { text: 'loading templates', indent: 8 }, + verbose + ).start(); await createOutputDir({ language, testPath }); @@ -83,7 +89,7 @@ export async function generateClientTests( }); if (!template) { - spinner.warn("Skipping because template doesn't exist"); + spinner.warn("skipping because template doesn't exist"); return; } diff --git a/scripts/cts/generate.ts b/scripts/cts/generate.ts index b1349a027c..286fb93b3b 100644 --- a/scripts/cts/generate.ts +++ b/scripts/cts/generate.ts @@ -1,3 +1,4 @@ +import { toAbsolutePath } from '../common'; import { getTestOutputFolder } from '../config'; import { formatter } from '../formatter'; import { createSpinner } from '../oraLog'; @@ -31,6 +32,10 @@ export async function ctsGenerateMany( const langs = [...new Set(generators.map((gen) => gen.language))]; for (const lang of langs) { - await formatter(lang, getTestOutputFolder(lang), verbose); + await formatter( + lang, + toAbsolutePath(`tests/output/${lang}/${getTestOutputFolder(lang)}`), + verbose + ); } } diff --git a/scripts/cts/methods/requests/cts.ts b/scripts/cts/methods/requests/cts.ts index 0214c62c58..06e9e0599d 100644 --- a/scripts/cts/methods/requests/cts.ts +++ b/scripts/cts/methods/requests/cts.ts @@ -3,11 +3,10 @@ import fsp from 'fs/promises'; import SwaggerParser from '@apidevtools/swagger-parser'; import type { OpenAPIV3 } from 'openapi-types'; -import { getAbsolutePath } from '../../../common'; +import { exists, toAbsolutePath } from '../../../common'; import { removeEnumType, removeObjectName, walk } from '../../utils'; import type { - CTS, CTSBlock, ParametersWithDataType, RequestCTS, @@ -94,14 +93,6 @@ function transformParam({ } return out; } - - if (!objectName) { - // throw new Error(`Object ${key} missing property $objectName in test ${testName}`); - // eslint-disable-next-line no-console - console.log( - `Object ${key} missing property $objectName in test ${testName}` - ); - } } else if (isArray) { // recursive on all value out = value.map((v, i) => @@ -142,14 +133,18 @@ function createParamWithDataType({ return [transformed]; } -async function loadRequestsCTS(client: string): Promise { +export async function loadRequestsCTS(client: string): Promise { // load the list of operations from the spec const spec = await SwaggerParser.validate( - getAbsolutePath(`specs/${client}/spec.yml`) + toAbsolutePath(`specs/${client}/spec.yml`) ); if (!spec.paths) { throw new Error(`No paths found for spec ${client}/spec.yml`); } + if (!(await exists(toAbsolutePath(`tests/CTS/methods/requests/${client}`)))) { + // skip it if no CTS for this client + return []; + } const operations = Object.values(spec.paths) .flatMap((p) => Object.values(p)) @@ -158,7 +153,7 @@ async function loadRequestsCTS(client: string): Promise { const ctsClient: CTSBlock[] = []; for await (const file of walk( - getAbsolutePath(`tests/CTS/methods/requests/${client}`) + toAbsolutePath(`tests/CTS/methods/requests/${client}`) )) { if (!file.name.endsWith('json')) { continue; @@ -228,9 +223,3 @@ async function loadRequestsCTS(client: string): Promise { t1.operationId.localeCompare(t2.operationId) ); } - -export async function loadCTS(client: string): Promise { - return { - requests: await loadRequestsCTS(client), - }; -} diff --git a/scripts/cts/methods/requests/generate.ts b/scripts/cts/methods/requests/generate.ts index dd55d7383e..9cb1591375 100644 --- a/scripts/cts/methods/requests/generate.ts +++ b/scripts/cts/methods/requests/generate.ts @@ -12,7 +12,7 @@ import { loadTemplates, } from '../../utils'; -import { loadCTS } from './cts'; +import { loadRequestsCTS } from './cts'; const testPath = 'methods/requests'; @@ -24,21 +24,28 @@ export async function generateRequestsTests( }: Generator, verbose: boolean ): Promise { - createSpinner('generating requests tests', verbose).start().info(); - const spinner = createSpinner('loading templates', verbose).start(); + createSpinner({ text: 'generating requests tests', indent: 4 }, verbose) + .start() + .info(); + const spinner = createSpinner( + { text: 'loading templates', indent: 8 }, + verbose + ).start(); const { requests: template, ...partialTemplates } = await loadTemplates({ language, testPath, }); spinner.text = 'loading CTS'; - const cts = (await loadCTS(client)).requests; - await createOutputDir({ language, testPath }); + const cts = await loadRequestsCTS(client); if (cts.length === 0) { + spinner.warn("skipping because tests doesn't exist"); return; } + await createOutputDir({ language, testPath }); + spinner.text = 'rendering templates'; const code = Mustache.render( template, diff --git a/scripts/cts/utils.ts b/scripts/cts/utils.ts index e4298f01a1..e6ace617ab 100644 --- a/scripts/cts/utils.ts +++ b/scripts/cts/utils.ts @@ -1,7 +1,7 @@ import fsp from 'fs/promises'; import path from 'path'; -import { exists, getAbsolutePath } from '../common'; +import { exists, toAbsolutePath } from '../common'; import { getTestExtension, getTestOutputFolder } from '../config'; export async function* walk( @@ -76,7 +76,7 @@ export async function createOutputDir({ testPath: string; }): Promise { await fsp.mkdir( - getAbsolutePath( + toAbsolutePath( `tests/output/${language}/${getTestOutputFolder(language)}/${testPath}` ), { @@ -94,7 +94,7 @@ export function getOutputPath({ client: string; testPath: string; }): string { - return getAbsolutePath( + return toAbsolutePath( `tests/output/${language}/${getTestOutputFolder( language )}/${testPath}/${client}${getTestExtension(language)}` @@ -109,8 +109,8 @@ export async function loadTemplates({ testPath: string; }): Promise> { const templates: Record = {}; - const templatePath = getAbsolutePath( - `./CTS/${testPath}/templates/${language}` + const templatePath = toAbsolutePath( + `tests/CTS/${testPath}/templates/${language}` ); if (!(await exists(templatePath))) { diff --git a/scripts/formatter.ts b/scripts/formatter.ts index 55a543433c..0c72d9bfc4 100644 --- a/scripts/formatter.ts +++ b/scripts/formatter.ts @@ -6,7 +6,10 @@ export async function formatter( folder: string, verbose = false ): Promise { - const spinner = createSpinner(`formatting ${language}`, verbose).start(); + const spinner = createSpinner( + { text: `formatting ${language}`, indent: 4 }, + verbose + ).start(); let cmd = ''; switch (language) { case 'javascript': diff --git a/scripts/oraLog.ts b/scripts/oraLog.ts index 412ef0ccf3..6bc9ed2da8 100644 --- a/scripts/oraLog.ts +++ b/scripts/oraLog.ts @@ -1,11 +1,17 @@ -/* eslint-disable no-console */ import ora from 'ora-classic'; +type OraLogOptions = { text?: string; indent?: number }; class OraLog { private _text: string; + private _indent = 0; - constructor(text: string) { - this._text = text; + constructor(options: OraLogOptions | string) { + if (typeof options === 'string') { + this._text = options; + } else { + this._text = options.text ?? ''; + this._indent = options.indent ?? 0; + } } private maybeText(text?: string): void { @@ -16,7 +22,8 @@ class OraLog { } start(): this { - console.log(this._text); + // eslint-disable-next-line no-console + console.log(' '.repeat(this._indent) + this._text); return this; } @@ -44,17 +51,25 @@ class OraLog { this._text = text; this.start(); } + + get indent(): number { + return this._indent; + } + + set indent(indent: number) { + this._indent = indent; + } } /** * Returns a spinner that will log directly in verbose mode, to avoid conflict with other log. */ export function createSpinner( - text: string, + options: OraLogOptions | string, verbose: boolean ): ora.Ora | OraLog { if (verbose) { - return new OraLog(text); + return new OraLog(options); } - return ora(text); + return ora(options); } diff --git a/scripts/release/process-release.ts b/scripts/release/process-release.ts index 80111af269..3dec99ed08 100755 --- a/scripts/release/process-release.ts +++ b/scripts/release/process-release.ts @@ -5,7 +5,7 @@ import dotenv from 'dotenv'; import execa from 'execa'; import openapitools from '../../openapitools.json'; -import { getAbsolutePath, run } from '../common'; +import { toAbsolutePath, run } from '../common'; import { MAIN_BRANCH, OWNER, REPO, getMarkdownSection } from './common'; import TEXT from './text'; @@ -73,14 +73,14 @@ async function processRelease(): Promise { } }); fs.writeFileSync( - getAbsolutePath('openapitools.json'), + toAbsolutePath('openapitools.json'), JSON.stringify(openapitools, null, 2) ); // update changelogs new Set([...Object.keys(versionsToRelease), ...langsToUpdateRepo]).forEach( (lang) => { - const filePath = getAbsolutePath(`docs/changelogs/${lang}.md`); + const filePath = toAbsolutePath(`docs/changelogs/${lang}.md`); const header = versionsToRelease[lang!] ? `## ${versionsToRelease[lang!].next}` : `## ${new Date().toISOString().split('T')[0]}`; @@ -118,7 +118,7 @@ async function processRelease(): Promise { await run(`yarn api generate ${lang} all`, { verbose: true }); } - const clientPath = getAbsolutePath( + const clientPath = toAbsolutePath( 'clients/dummy-algoliasearch-client-javascript' ); const runInClient = (command: string, options = {}): Promise => From 8d52ccff3318937f4aba62eaa1a6aeda373d05c2 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Thu, 24 Feb 2022 17:58:00 +0100 Subject: [PATCH 13/20] skip cts if no config --- openapitools.json | 13 ++++++++++--- scripts/config.ts | 8 ++++---- scripts/cts/generate.ts | 6 ++++++ 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/openapitools.json b/openapitools.json index 2e82b8a813..9ed8a7f65f 100644 --- a/openapitools.json +++ b/openapitools.json @@ -260,7 +260,8 @@ "configClassname": "SearchConfig", "useCache": true, "variableNamingConvention": "camelCase", - "packageName": "algoliasearch-client-php" + "packageName": "algoliasearch-client-php", + "packageVersion": "0.0.1" } }, "php-recommend": { @@ -278,7 +279,8 @@ "configClassname": "RecommendConfig", "useCache": true, "variableNamingConvention": "camelCase", - "packageName": "algoliasearch-client-php" + "packageName": "algoliasearch-client-php", + "packageVersion": "0.0.1" } }, "php-personalization": { @@ -298,6 +300,7 @@ "allowedRegions": "us-eu", "variableNamingConvention": "camelCase", "packageName": "algoliasearch-client-php", + "packageVersion": "0.0.1", "isEuHost": true, "host": "personalization", "topLevelDomain": "com" @@ -320,6 +323,7 @@ "allowedRegions": "us-de", "variableNamingConvention": "camelCase", "packageName": "algoliasearch-client-php", + "packageVersion": "0.0.1", "fallbackToAliasHost": true, "isDeHost": true, "host": "analytics", @@ -343,6 +347,7 @@ "allowedRegions": "us-de", "variableNamingConvention": "camelCase", "packageName": "algoliasearch-client-php", + "packageVersion": "0.0.1", "fallbackToAliasHost": true, "isDeHost": true, "host": "insights", @@ -366,6 +371,7 @@ "allowedRegions": "us-de", "variableNamingConvention": "camelCase", "packageName": "algoliasearch-client-php", + "packageVersion": "0.0.1", "fallbackToAliasHost": true, "isDeHost": true, "host": "analytics", @@ -389,6 +395,7 @@ "allowedRegions": "us-eu", "variableNamingConvention": "camelCase", "packageName": "algoliasearch-client-php", + "packageVersion": "0.0.1", "isEuHost": true, "host": "query-suggestions", "topLevelDomain": "com" @@ -396,4 +403,4 @@ } } } -} \ No newline at end of file +} diff --git a/scripts/config.ts b/scripts/config.ts index c83a43911f..25baddcb41 100644 --- a/scripts/config.ts +++ b/scripts/config.ts @@ -4,10 +4,10 @@ export function getLanguageFolder(language: string): string { return clientsConfig[language].folder; } -export function getTestExtension(language: string): string { - return clientsConfig[language].tests.extension; +export function getTestExtension(language: string): string | undefined { + return clientsConfig[language]?.tests?.extension; } -export function getTestOutputFolder(language: string): string { - return clientsConfig[language].tests.outputFolder; +export function getTestOutputFolder(language: string): string | undefined { + return clientsConfig[language]?.tests?.outputFolder; } diff --git a/scripts/cts/generate.ts b/scripts/cts/generate.ts index 286fb93b3b..e2b416835c 100644 --- a/scripts/cts/generate.ts +++ b/scripts/cts/generate.ts @@ -27,11 +27,17 @@ export async function ctsGenerateMany( verbose: boolean ): Promise { for (const gen of generators) { + if (!getTestOutputFolder(gen.language)) { + continue; + } await ctsGenerate(gen, verbose); } const langs = [...new Set(generators.map((gen) => gen.language))]; for (const lang of langs) { + if (!getTestOutputFolder(lang)) { + continue; + } await formatter( lang, toAbsolutePath(`tests/output/${lang}/${getTestOutputFolder(lang)}`), From a24f5526234c2e9ae464027901ed5efa46f898d1 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Thu, 24 Feb 2022 18:10:55 +0100 Subject: [PATCH 14/20] running cts --- package.json | 3 +- scripts/cts/run.ts | 22 ++++++--- scripts/index.ts | 6 ++- tests/output/javascript/tsconfig.json | 5 +- yarn.lock | 69 ++++++++++++++++++++++++++- 5 files changed, 94 insertions(+), 11 deletions(-) diff --git a/package.json b/package.json index b57ea1185d..2c269a2a15 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,8 @@ "clients/algoliasearch-client-javascript/", "playground/javascript/node/", "playground/javascript/browser/", - "scripts/" + "scripts/", + "tests/output/javascript" ], "scripts": { "api": "yarn workspace scripts ts-node --transpile-only ./index.ts", diff --git a/scripts/cts/run.ts b/scripts/cts/run.ts index 861ccd525d..e27ee26243 100644 --- a/scripts/cts/run.ts +++ b/scripts/cts/run.ts @@ -1,9 +1,8 @@ import { CI, run } from '../common'; +import { createSpinner } from '../oraLog'; -export async function runCts( - language: string, - verbose: boolean -): Promise { +async function runCtsOne(language: string, verbose: boolean): Promise { + const spinner = createSpinner(`running cts for ${language}`, verbose).start(); switch (language) { case 'javascript': await run('yarn workspace javascript-tests test', { verbose }); @@ -18,12 +17,23 @@ export async function runCts( if (CI) php = 'php'; await run( - `${php} ./clients/algoliasearch-client-php/vendor/bin/phpunit ./`, + `${php} ./clients/algoliasearch-client-php/vendor/bin/phpunit tests/output/php`, { verbose } ); break; } default: - // echo "Skipping unknown language $LANGUAGE to run the CTS" + spinner.warn(`skipping unknown language ${language} to run the CTS`); + return; + } + spinner.succeed(); +} + +export async function runCts( + languages: string[], + verbose: boolean +): Promise { + for (const lang of languages) { + await runCtsOne(lang, verbose); } } diff --git a/scripts/index.ts b/scripts/index.ts index c8e4cc9779..9945a35a4e 100644 --- a/scripts/index.ts +++ b/scripts/index.ts @@ -224,7 +224,11 @@ ctsCommand .action(async (language: string | undefined, { verbose }) => { language = await promptLanguage(language); - await runCts(language, Boolean(verbose)); + let langsTodo = [language]; + if (language === 'all') { + langsTodo = LANGUAGES; + } + await runCts(langsTodo, Boolean(verbose)); }); program diff --git a/tests/output/javascript/tsconfig.json b/tests/output/javascript/tsconfig.json index 27113bfb38..c5cb759b73 100644 --- a/tests/output/javascript/tsconfig.json +++ b/tests/output/javascript/tsconfig.json @@ -1,9 +1,10 @@ { - "extends": "../../tsconfig.json", + "extends": "../../../base.tsconfig.json", "compilerOptions": { "typeRoots": ["../../../node_modules/@types"], + "types": ["node", "jest"], "outDir": "dist" }, "include": ["src"], - "exclude": [] + "exclude": ["dist", "node_modules"] } diff --git a/yarn.lock b/yarn.lock index eb5bda0126..5dfc6e4078 100644 --- a/yarn.lock +++ b/yarn.lock @@ -65,7 +65,7 @@ __metadata: languageName: unknown linkType: soft -"@algolia/client-insights@workspace:clients/algoliasearch-client-javascript/packages/client-insights": +"@algolia/client-insights@5.0.0, @algolia/client-insights@workspace:clients/algoliasearch-client-javascript/packages/client-insights": version: 0.0.0-use.local resolution: "@algolia/client-insights@workspace:clients/algoliasearch-client-javascript/packages/client-insights" dependencies: @@ -3134,6 +3134,16 @@ __metadata: languageName: node linkType: hard +"@types/jest@npm:27.0.3": + version: 27.0.3 + resolution: "@types/jest@npm:27.0.3" + dependencies: + jest-diff: ^27.0.0 + pretty-format: ^27.0.0 + checksum: 3683a9945821966f6dccddf337219a5d682633687c9d30df859223db553589f63e9b2c34e69f0cc845c86ffcf115742f25c12ea03c8d33d2244890fdc0af61e2 + languageName: node + linkType: hard + "@types/jest@npm:27.4.0": version: 27.4.0 resolution: "@types/jest@npm:27.4.0" @@ -7103,6 +7113,29 @@ __metadata: languageName: unknown linkType: soft +"javascript-tests@workspace:tests/output/javascript": + version: 0.0.0-use.local + resolution: "javascript-tests@workspace:tests/output/javascript" + dependencies: + "@algolia/client-abtesting": 5.0.0 + "@algolia/client-analytics": 5.0.0 + "@algolia/client-common": 5.0.0 + "@algolia/client-insights": 5.0.0 + "@algolia/client-personalization": 5.0.0 + "@algolia/client-query-suggestions": 5.0.0 + "@algolia/client-search": 5.0.0 + "@algolia/client-sources": 0.0.1 + "@algolia/recommend": 5.0.0 + "@algolia/requester-node-http": 5.0.0 + "@types/jest": 27.0.3 + "@types/node": 16.11.11 + jest: 27.4.7 + ts-jest: 27.1.2 + ts-node: 10.5.0 + typescript: 4.5.4 + languageName: unknown + linkType: soft + "jest-changed-files@npm:^27.5.1": version: 27.5.1 resolution: "jest-changed-files@npm:27.5.1" @@ -10869,6 +10902,40 @@ __metadata: languageName: node linkType: hard +"ts-jest@npm:27.1.2": + version: 27.1.2 + resolution: "ts-jest@npm:27.1.2" + dependencies: + bs-logger: 0.x + fast-json-stable-stringify: 2.x + jest-util: ^27.0.0 + json5: 2.x + lodash.memoize: 4.x + make-error: 1.x + semver: 7.x + yargs-parser: 20.x + peerDependencies: + "@babel/core": ">=7.0.0-beta.0 <8" + "@types/jest": ^27.0.0 + babel-jest: ">=27.0.0 <28" + esbuild: ~0.14.0 + jest: ^27.0.0 + typescript: ">=3.8 <5.0" + peerDependenciesMeta: + "@babel/core": + optional: true + "@types/jest": + optional: true + babel-jest: + optional: true + esbuild: + optional: true + bin: + ts-jest: cli.js + checksum: 2e7275f8a3545ec1340b37c458ace9244b5903e86861eb108beffff97d433f296c1254f76a41b573b1fe6245110b21bb62150bb88d55159f1dc7a929886535cb + languageName: node + linkType: hard + "ts-jest@npm:27.1.3": version: 27.1.3 resolution: "ts-jest@npm:27.1.3" From 52194da43416e79c6bfe296f5e420fdd0f2a47f7 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Thu, 24 Feb 2022 18:22:55 +0100 Subject: [PATCH 15/20] simple log for CI --- scripts/cts/{run.ts => runCts.ts} | 6 +++--- scripts/index.ts | 2 +- scripts/oraLog.ts | 4 +++- 3 files changed, 7 insertions(+), 5 deletions(-) rename scripts/cts/{run.ts => runCts.ts} (93%) diff --git a/scripts/cts/run.ts b/scripts/cts/runCts.ts similarity index 93% rename from scripts/cts/run.ts rename to scripts/cts/runCts.ts index e27ee26243..34bbb0ace9 100644 --- a/scripts/cts/run.ts +++ b/scripts/cts/runCts.ts @@ -1,4 +1,4 @@ -import { CI, run } from '../common'; +import { run } from '../common'; import { createSpinner } from '../oraLog'; async function runCtsOne(language: string, verbose: boolean): Promise { @@ -12,16 +12,16 @@ async function runCtsOne(language: string, verbose: boolean): Promise { verbose, }); break; + /* not working yet case 'php': { let php = 'php8'; if (CI) php = 'php'; - await run( `${php} ./clients/algoliasearch-client-php/vendor/bin/phpunit tests/output/php`, { verbose } ); break; - } + }*/ default: spinner.warn(`skipping unknown language ${language} to run the CTS`); return; diff --git a/scripts/index.ts b/scripts/index.ts index 9945a35a4e..802f55c198 100644 --- a/scripts/index.ts +++ b/scripts/index.ts @@ -14,7 +14,7 @@ import { LANGUAGES, } from './common'; import { ctsGenerateMany } from './cts/generate'; -import { runCts } from './cts/run'; +import { runCts } from './cts/runCts'; import { generate } from './generate'; import { playground } from './playground'; import type { Generator } from './types'; diff --git a/scripts/oraLog.ts b/scripts/oraLog.ts index 6bc9ed2da8..20ece0f229 100644 --- a/scripts/oraLog.ts +++ b/scripts/oraLog.ts @@ -1,5 +1,7 @@ import ora from 'ora-classic'; +import { CI } from './common'; + type OraLogOptions = { text?: string; indent?: number }; class OraLog { private _text: string; @@ -68,7 +70,7 @@ export function createSpinner( options: OraLogOptions | string, verbose: boolean ): ora.Ora | OraLog { - if (verbose) { + if (verbose || CI) { return new OraLog(options); } return ora(options); From 5502d8bdac93285965bbede068936821cf2bb746 Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Thu, 24 Feb 2022 18:30:36 +0100 Subject: [PATCH 16/20] toAbsolutePath --- scripts/buildSpecs.ts | 12 ++++++------ scripts/pre-gen/setHostsOptions.ts | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/buildSpecs.ts b/scripts/buildSpecs.ts index 18994e3cc2..3575d58db3 100644 --- a/scripts/buildSpecs.ts +++ b/scripts/buildSpecs.ts @@ -2,7 +2,7 @@ import fsp from 'fs/promises'; import { hashElement } from 'folder-hash'; -import { exists, run } from './common'; +import { exists, run, toAbsolutePath } from './common'; import { createSpinner } from './oraLog'; async function buildSpec( @@ -11,15 +11,15 @@ async function buildSpec( verbose: boolean, useCache: boolean ): Promise { - const cacheFile = `../specs/dist/${client}.cache`; + const cacheFile = toAbsolutePath(`specs/dist/${client}.cache`); if (useCache) { // check if file and cache exist const spinner = createSpinner( `checking cache for ${client}`, verbose ).start(); - if (await exists(`../specs/bundled/${client}.yml`)) { - const hash = (await hashElement(`../specs/${client}`)).hash; + if (await exists(toAbsolutePath(`specs/bundled/${client}.yml`))) { + const hash = (await hashElement(toAbsolutePath(`specs/${client}`))).hash; // compare with stored cache if (await exists(cacheFile)) { const storedHash = (await fsp.readFile(cacheFile)).toString(); @@ -50,7 +50,7 @@ async function buildSpec( }); spinner.text = `storing ${client} cache`; - const hash = (await hashElement(`../specs/${client}`)).hash; + const hash = (await hashElement(toAbsolutePath(`specs/${client}`))).hash; await fsp.writeFile(cacheFile, hash); spinner.succeed(); @@ -62,7 +62,7 @@ export async function buildSpecs( verbose: boolean, useCache: boolean ): Promise { - await fsp.mkdir('../specs/dist', { recursive: true }); + await fsp.mkdir(toAbsolutePath('specs/dist'), { recursive: true }); for (const client of clients) { await buildSpec(client, outputFormat, verbose, useCache); diff --git a/scripts/pre-gen/setHostsOptions.ts b/scripts/pre-gen/setHostsOptions.ts index d7ff3c15c7..e60fea2063 100644 --- a/scripts/pre-gen/setHostsOptions.ts +++ b/scripts/pre-gen/setHostsOptions.ts @@ -1,9 +1,9 @@ import { readFile, stat, writeFile } from 'fs/promises'; -import path from 'path'; import { URL } from 'url'; import yaml from 'js-yaml'; +import { toAbsolutePath } from '../common'; import type { Generator } from '../types'; type Server = { @@ -37,7 +37,7 @@ export async function setHostsOptions({ client, key: generator, }: Pick): Promise { - const openapitoolsPath = path.join(process.cwd(), '../openapitools.json'); + const openapitoolsPath = toAbsolutePath('openapitools.json'); if (!(await stat(openapitoolsPath))) { throw new Error( `File not found ${openapitoolsPath}.\nMake sure your run scripts from the root directory using yarn workspace.` @@ -51,7 +51,7 @@ export async function setHostsOptions({ throw new Error(`Generator not found: ${generator}`); } - const specPath = path.join(process.cwd(), `../specs/bundled/${client}.yml`); + const specPath = toAbsolutePath(`specs/bundled/${client}.yml`); if (!(await stat(specPath))) { throw new Error( From aff4a813f8540561e1c3a10b4483857753a2b2ed Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Fri, 25 Feb 2022 09:44:02 +0100 Subject: [PATCH 17/20] remove deprecated dep --- .github/workflows/check.yml | 4 +- package.json | 8 +-- scripts/package.json | 1 - yarn.lock | 118 +----------------------------------- 4 files changed, 8 insertions(+), 123 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 8520105168..159f30fe89 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -238,7 +238,7 @@ jobs: uses: ./.github/actions/cache - name: Check script linting - run: yarn lint:scripts + run: yarn scripts:lint - name: Test scripts - run: yarn test:scripts + run: yarn scripts:test diff --git a/package.json b/package.json index 2c269a2a15..77a807ea38 100644 --- a/package.json +++ b/package.json @@ -17,12 +17,12 @@ "docker:setup": "yarn docker:clean && yarn docker:build && yarn docker:mount", "docker": "docker exec -it dev yarn api $*", "github-actions:lint": "eslint --ext=yml .github/", - "lint:scripts": "eslint --ext=ts scripts/", "playground:browser": "yarn workspace javascript-browser-playground start", - "specs:fix": "eslint --ext=yml specs/ --fix", "release": "yarn workspace scripts createReleaseIssue", - "specs:lint": "eslint --ext=yml specs/$0", - "test:scripts": "yarn workspace scripts test" + "scripts:lint": "eslint --ext=ts scripts/", + "scripts:test": "yarn workspace scripts test", + "specs:fix": "eslint --ext=yml specs/ --fix", + "specs:lint": "eslint --ext=yml specs/$0" }, "devDependencies": { "@openapitools/openapi-generator-cli": "2.4.26", diff --git a/scripts/package.json b/scripts/package.json index deb90a4d6e..e48daed9bb 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -16,7 +16,6 @@ "@types/js-yaml": "4.0.5", "@types/mustache": "4.1.2", "@types/node": "16.11.11", - "@types/ora": "3.2.0", "@types/semver": "7.3.9", "commander": "9.0.0", "dotenv": "16.0.0", diff --git a/yarn.lock b/yarn.lock index 5dfc6e4078..e7e2b440aa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3210,15 +3210,6 @@ __metadata: languageName: node linkType: hard -"@types/ora@npm:3.2.0": - version: 3.2.0 - resolution: "@types/ora@npm:3.2.0" - dependencies: - ora: "*" - checksum: dba266e595627c78aee8e4d2dd7f552c3389f520d27b6474164bf0180a7941c72618b02ef4efcbb02e0a8ae43faf407f9c67ffe35760101fe6b1290bbe46158b - languageName: node - linkType: hard - "@types/parse-json@npm:^4.0.0": version: 4.0.0 resolution: "@types/parse-json@npm:4.0.0" @@ -3603,13 +3594,6 @@ __metadata: languageName: node linkType: hard -"ansi-regex@npm:^6.0.1": - version: 6.0.1 - resolution: "ansi-regex@npm:6.0.1" - checksum: 1ff8b7667cded1de4fa2c9ae283e979fc87036864317da86a2e546725f96406746411d0d85e87a2d12fa5abd715d90006de7fa4fa0477c92321ad3b4c7d4e169 - languageName: node - linkType: hard - "ansi-styles@npm:^3.1.0, ansi-styles@npm:^3.2.1": version: 3.2.1 resolution: "ansi-styles@npm:3.2.1" @@ -4011,17 +3995,6 @@ __metadata: languageName: node linkType: hard -"bl@npm:^5.0.0": - version: 5.0.0 - resolution: "bl@npm:5.0.0" - dependencies: - buffer: ^6.0.3 - inherits: ^2.0.4 - readable-stream: ^3.4.0 - checksum: 5dbbcf9cbcf55221dc21f48968bc8cd6d78faea3c653d496ff8e0c382b95e8b6c4b9e818fe67de2f97ed0cd0c219c350ccce42aca91be33e0ad12e698c615061 - languageName: node - linkType: hard - "boolbase@npm:^1.0.0": version: 1.0.0 resolution: "boolbase@npm:1.0.0" @@ -4156,16 +4129,6 @@ __metadata: languageName: node linkType: hard -"buffer@npm:^6.0.3": - version: 6.0.3 - resolution: "buffer@npm:6.0.3" - dependencies: - base64-js: ^1.3.1 - ieee754: ^1.2.1 - checksum: 5ad23293d9a731e4318e420025800b42bf0d264004c0286c8cc010af7a270c7a0f6522e84f54b9ad65cbd6db20b8badbfd8d2ebf4f80fa03dab093b89e68c3f9 - languageName: node - linkType: hard - "builtin-modules@npm:^3.1.0": version: 3.2.0 resolution: "builtin-modules@npm:3.2.0" @@ -4356,13 +4319,6 @@ __metadata: languageName: node linkType: hard -"chalk@npm:^5.0.0": - version: 5.0.0 - resolution: "chalk@npm:5.0.0" - checksum: 6eba7c518b9aa5fe882ae6d14a1ffa58c418d72a3faa7f72af56641f1bbef51b645fca1d6e05d42357b7d3c846cd504c0b7b64d12309cdd07867e3b4411e0d01 - languageName: node - linkType: hard - "char-regex@npm:^1.0.2": version: 1.0.2 resolution: "char-regex@npm:1.0.2" @@ -4470,16 +4426,7 @@ __metadata: languageName: node linkType: hard -"cli-cursor@npm:^4.0.0": - version: 4.0.0 - resolution: "cli-cursor@npm:4.0.0" - dependencies: - restore-cursor: ^4.0.0 - checksum: ab3f3ea2076e2176a1da29f9d64f72ec3efad51c0960898b56c8a17671365c26e67b735920530eaf7328d61f8bd41c27f46b9cf6e4e10fe2fa44b5e8c0e392cc - languageName: node - linkType: hard - -"cli-spinners@npm:^2.5.0, cli-spinners@npm:^2.6.1": +"cli-spinners@npm:^2.5.0": version: 2.6.1 resolution: "cli-spinners@npm:2.6.1" checksum: 423409baaa7a58e5104b46ca1745fbfc5888bbd0b0c5a626e052ae1387060839c8efd512fb127e25769b3dc9562db1dc1b5add6e0b93b7ef64f477feb6416a45 @@ -6580,7 +6527,7 @@ __metadata: languageName: node linkType: hard -"ieee754@npm:^1.1.13, ieee754@npm:^1.2.1": +"ieee754@npm:^1.1.13": version: 1.2.1 resolution: "ieee754@npm:1.2.1" checksum: 5144c0c9815e54ada181d80a0b810221a253562422e7c6c3a60b1901154184f49326ec239d618c416c1c5945a2e197107aee8d986a3dd836b53dffefd99b5e7e @@ -6854,13 +6801,6 @@ __metadata: languageName: node linkType: hard -"is-interactive@npm:^2.0.0": - version: 2.0.0 - resolution: "is-interactive@npm:2.0.0" - checksum: e8d52ad490bed7ae665032c7675ec07732bbfe25808b0efbc4d5a76b1a1f01c165f332775c63e25e9a03d319ebb6b24f571a9e902669fc1e40b0a60b5be6e26c - languageName: node - linkType: hard - "is-json@npm:^2.0.1": version: 2.0.1 resolution: "is-json@npm:2.0.1" @@ -6975,13 +6915,6 @@ __metadata: languageName: node linkType: hard -"is-unicode-supported@npm:^1.1.0": - version: 1.1.0 - resolution: "is-unicode-supported@npm:1.1.0" - checksum: 1f2504d94383ea180ea25e729b40b1d97398a37325c2e62db96a3e98b457c767259dd5bbf9ab2815e83e5012dc4b61d533e75d12df7f208c470474d821bd5f24 - languageName: node - linkType: hard - "is-weakref@npm:^1.0.1": version: 1.0.2 resolution: "is-weakref@npm:1.0.2" @@ -8020,16 +7953,6 @@ __metadata: languageName: node linkType: hard -"log-symbols@npm:^5.1.0": - version: 5.1.0 - resolution: "log-symbols@npm:5.1.0" - dependencies: - chalk: ^5.0.0 - is-unicode-supported: ^1.1.0 - checksum: 7291b6e7f1b3df6865bdaeb9b59605c832668ac2fa0965c63b1e7dd3700349aec09c1d7d40c368d5041ff58b7f89461a56e4009471921301af7b3609cbff9a29 - languageName: node - linkType: hard - "lru-cache@npm:^6.0.0": version: 6.0.0 resolution: "lru-cache@npm:6.0.0" @@ -8792,23 +8715,6 @@ __metadata: languageName: node linkType: hard -"ora@npm:*": - version: 6.1.0 - resolution: "ora@npm:6.1.0" - dependencies: - bl: ^5.0.0 - chalk: ^5.0.0 - cli-cursor: ^4.0.0 - cli-spinners: ^2.6.1 - is-interactive: ^2.0.0 - is-unicode-supported: ^1.1.0 - log-symbols: ^5.1.0 - strip-ansi: ^7.0.1 - wcwidth: ^1.0.1 - checksum: 0e84d9c32f2c62617324658ea547963412152f0bf60c1580ed18ec660769713452a6a47d117a1767bb118bb0867720ecc109ef6e12d18fd71ae244683e722b23 - languageName: node - linkType: hard - "ora@npm:^5.4.1": version: 5.4.1 resolution: "ora@npm:5.4.1" @@ -9978,16 +9884,6 @@ __metadata: languageName: node linkType: hard -"restore-cursor@npm:^4.0.0": - version: 4.0.0 - resolution: "restore-cursor@npm:4.0.0" - dependencies: - onetime: ^5.1.0 - signal-exit: ^3.0.2 - checksum: 5b675c5a59763bf26e604289eab35711525f11388d77f409453904e1e69c0d37ae5889295706b2c81d23bd780165084d040f9b68fffc32cc921519031c4fa4af - languageName: node - linkType: hard - "retry@npm:^0.12.0": version: 0.12.0 resolution: "retry@npm:0.12.0" @@ -10194,7 +10090,6 @@ __metadata: "@types/js-yaml": 4.0.5 "@types/mustache": 4.1.2 "@types/node": 16.11.11 - "@types/ora": 3.2.0 "@types/semver": 7.3.9 commander: 9.0.0 dotenv: 16.0.0 @@ -10585,15 +10480,6 @@ __metadata: languageName: node linkType: hard -"strip-ansi@npm:^7.0.1": - version: 7.0.1 - resolution: "strip-ansi@npm:7.0.1" - dependencies: - ansi-regex: ^6.0.1 - checksum: 257f78fa433520e7f9897722731d78599cb3fce29ff26a20a5e12ba4957463b50a01136f37c43707f4951817a75e90820174853d6ccc240997adc5df8f966039 - languageName: node - linkType: hard - "strip-bom@npm:^3.0.0": version: 3.0.0 resolution: "strip-bom@npm:3.0.0" From 02d6a9fa6624ff6c82726ef1eabad403921eb5ce Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Fri, 25 Feb 2022 11:18:25 +0100 Subject: [PATCH 18/20] review --- .github/workflows/check.yml | 18 ++++---- README.md | 4 +- openapitools.json | 2 +- package.json | 4 +- scripts/buildClients.ts | 26 ++++++----- scripts/buildSpecs.ts | 19 +++++--- scripts/generate.ts | 4 +- scripts/index.ts | 73 ++++++++++++++++++------------ scripts/release/process-release.ts | 4 +- 9 files changed, 90 insertions(+), 64 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 159f30fe89..c1fb46f3b7 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -60,7 +60,7 @@ jobs: uses: ./.github/actions/cache - name: Building ${{ matrix.client }} specs - run: yarn api build specs ${{ matrix.client }} + run: yarn cli build specs ${{ matrix.client }} - name: Check diff with pushed spec run: exit $(git status --porcelain specs/bundled/${{ matrix.client }}.yml | wc -l) @@ -123,7 +123,7 @@ jobs: - name: Generate ${{ matrix.client.name }} client if: steps.cache.outputs.cache-hit != 'true' - run: yarn api generate javascript ${{ matrix.client.name }} + run: yarn cli generate javascript ${{ matrix.client.name }} - name: Check diff with pushed client if: steps.cache.outputs.cache-hit != 'true' @@ -131,7 +131,7 @@ jobs: - name: Build ${{ matrix.client.name }} client if: steps.cache.outputs.cache-hit != 'true' - run: yarn api build clients javascript ${{ matrix.client.name }} + run: yarn cli build clients javascript ${{ matrix.client.name }} client_java: runs-on: ubuntu-20.04 @@ -160,7 +160,7 @@ jobs: - name: Generate ${{ matrix.client.name }} client if: steps.cache.outputs.cache-hit != 'true' - run: yarn api generate java ${{ matrix.client.name }} + run: yarn cli generate java ${{ matrix.client.name }} - name: Check diff with pushed client if: steps.cache.outputs.cache-hit != 'true' @@ -168,7 +168,7 @@ jobs: - name: Build ${{ matrix.client.name }} client if: steps.cache.outputs.cache-hit != 'true' - run: yarn api build clients java ${{ matrix.client.name }} + run: yarn cli build clients java ${{ matrix.client.name }} client_php: runs-on: ubuntu-20.04 @@ -190,7 +190,7 @@ jobs: - name: Generate ${{ matrix.client.name }} client if: steps.cache.outputs.cache-hit != 'true' - run: yarn api generate php ${{ matrix.client.name }} + run: yarn cli generate php ${{ matrix.client.name }} - name: Check diff with pushed client if: steps.cache.outputs.cache-hit != 'true' @@ -198,7 +198,7 @@ jobs: - name: Build ${{ matrix.client.name }} client if: steps.cache.outputs.cache-hit != 'true' - run: yarn api build clients php ${{ matrix.client.name }} + run: yarn cli build clients php ${{ matrix.client.name }} cts: runs-on: ubuntu-20.04 @@ -218,13 +218,13 @@ jobs: job: cts - name: Generate CTS - run: yarn api cts generate + run: yarn cli cts generate - name: Check diff with pushed CTS run: exit $(git status --porcelain ./tests/output | wc -l) - name: Run CTS - run: yarn api cts run + run: yarn cli cts run scripts: runs-on: ubuntu-20.04 diff --git a/README.md b/README.md index a989f587d5..069cc6044f 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ yarn docker build specs #### Build all specs ```bash -yarn docker build specs all +yarn docker build specs ``` #### Build specific spec @@ -94,7 +94,7 @@ yarn docker generate #### Generate all clients ```bash -yarn docker generate all all +yarn docker generate ``` ### Generate specific client for specific language diff --git a/openapitools.json b/openapitools.json index 9ed8a7f65f..c7f40ae154 100644 --- a/openapitools.json +++ b/openapitools.json @@ -403,4 +403,4 @@ } } } -} +} \ No newline at end of file diff --git a/package.json b/package.json index 77a807ea38..0d76fc72c0 100644 --- a/package.json +++ b/package.json @@ -9,13 +9,13 @@ "tests/output/javascript" ], "scripts": { - "api": "yarn workspace scripts ts-node --transpile-only ./index.ts", + "cli": "yarn workspace scripts ts-node --transpile-only ./index.ts", "clean": "rm -rf **/dist **/build **/node_modules **/.gradle", "docker:build": "./scripts/docker/build.sh", "docker:clean": "docker stop dev; docker rm -f dev; docker image rm -f api-clients-automation", "docker:mount": "./scripts/docker/mount.sh", "docker:setup": "yarn docker:clean && yarn docker:build && yarn docker:mount", - "docker": "docker exec -it dev yarn api $*", + "docker": "docker exec -it dev yarn cli $*", "github-actions:lint": "eslint --ext=yml .github/", "playground:browser": "yarn workspace javascript-browser-playground start", "release": "yarn workspace scripts createReleaseIssue", diff --git a/scripts/buildClients.ts b/scripts/buildClients.ts index 09e83d65f9..5facbf1189 100644 --- a/scripts/buildClients.ts +++ b/scripts/buildClients.ts @@ -48,18 +48,22 @@ export async function buildClients( generators: Generator[], verbose: boolean ): Promise { - for (const gen of generators) { - // we should detect this from the config file, for example `isMutliBuild` - if (gen.language === 'javascript') { - await buildPerClient(gen, verbose); - } - } + // For javascript, we build each client individually, + // we should detect this from the config file, for example `isMutliBuild` // for other languages, we can mutualize the build const langs = [...new Set(generators.map((gen) => gen.language))]; - for (const lang of langs) { - if (lang !== 'javascript') { - await buildAllClients(lang, verbose); - } - } + + await Promise.all([ + Promise.all( + generators + .filter(({ language }) => language === 'javascript') + .map((gen) => buildPerClient(gen, verbose)) + ), + Promise.all( + langs + .filter((lang) => lang !== 'javascript') + .map((lang) => buildAllClients(lang, verbose)) + ), + ]); } diff --git a/scripts/buildSpecs.ts b/scripts/buildSpecs.ts index 3575d58db3..1311928dfb 100644 --- a/scripts/buildSpecs.ts +++ b/scripts/buildSpecs.ts @@ -11,16 +11,17 @@ async function buildSpec( verbose: boolean, useCache: boolean ): Promise { + createSpinner(`${client} spec`, verbose).start().info(); const cacheFile = toAbsolutePath(`specs/dist/${client}.cache`); if (useCache) { - // check if file and cache exist const spinner = createSpinner( `checking cache for ${client}`, verbose ).start(); + // check if file and cache exist if (await exists(toAbsolutePath(`specs/bundled/${client}.yml`))) { - const hash = (await hashElement(toAbsolutePath(`specs/${client}`))).hash; // compare with stored cache + const hash = (await hashElement(toAbsolutePath(`specs/${client}`))).hash; if (await exists(cacheFile)) { const storedHash = (await fsp.readFile(cacheFile)).toString(); if (storedHash === hash) { @@ -44,16 +45,20 @@ async function buildSpec( { verbose } ); - spinner.text = `validate ${client} spec`; + spinner.text = `validating ${client} spec`; await run(`yarn openapi lint specs/bundled/${client}.${outputFormat}`, { verbose, }); - spinner.text = `storing ${client} cache`; + spinner.text = `storing ${client} spec cache`; const hash = (await hashElement(toAbsolutePath(`specs/${client}`))).hash; await fsp.writeFile(cacheFile, hash); spinner.succeed(); + + createSpinner(`building complete for ${client} spec`, verbose) + .start() + .succeed(); } export async function buildSpecs( @@ -64,7 +69,7 @@ export async function buildSpecs( ): Promise { await fsp.mkdir(toAbsolutePath('specs/dist'), { recursive: true }); - for (const client of clients) { - await buildSpec(client, outputFormat, verbose, useCache); - } + await Promise.all( + clients.map((client) => buildSpec(client, outputFormat, verbose, useCache)) + ); } diff --git a/scripts/generate.ts b/scripts/generate.ts index e68c982f68..5a6d5a594c 100644 --- a/scripts/generate.ts +++ b/scripts/generate.ts @@ -71,13 +71,13 @@ export async function generate( } if (lang === 'javascript') { const spinner = createSpinner( - 'Cleaning JavaScript client utils', + 'cleaning JavaScript client utils', verbose ).start(); await run('yarn workspace algoliasearch-client-javascript clean:utils', { verbose, }); - spinner.text = 'Building JavaScript client utils'; + spinner.text = 'building JavaScript client utils'; await run('yarn workspace algoliasearch-client-javascript build:utils', { verbose, }); diff --git a/scripts/index.ts b/scripts/index.ts index 802f55c198..8ccae24a1a 100644 --- a/scripts/index.ts +++ b/scripts/index.ts @@ -26,13 +26,16 @@ if (!CI && !DOCKER) { process.exit(1); } -program.name('api'); +program.name('cli'); -async function promptLanguage(defaut: string | undefined): Promise { +async function promptLanguage( + defaut: string | undefined, + interactive: boolean +): Promise { if (defaut) { return defaut; } - if (CI) { + if (!interactive) { return 'all'; } const { language } = await inquirer.prompt([ @@ -49,12 +52,13 @@ async function promptLanguage(defaut: string | undefined): Promise { async function promptClient( defaut: string | undefined, + interactive: boolean, clientList = CLIENTS ): Promise { if (defaut) { return defaut; } - if (CI) { + if (!interactive) { return 'all'; } const { client } = await inquirer.prompt([ @@ -106,14 +110,15 @@ program new Argument('[client]', 'The client').choices(['all'].concat(CLIENTS)) ) .option('-v, --verbose', 'make the generation verbose') + .option('-i, --interactive', 'open prompt to query parameters') .action( async ( language: string | undefined, client: string | undefined, - { verbose } + { verbose, interactive } ) => { - language = await promptLanguage(language); - client = await promptClient(client); + language = await promptLanguage(language, interactive); + client = await promptClient(client, interactive); await generate(generatorList({ language, client }), Boolean(verbose)); } @@ -133,14 +138,15 @@ buildCommand new Argument('[client]', 'The client').choices(['all'].concat(CLIENTS_JS)) ) .option('-v, --verbose', 'make the compilation verbose') + .option('-i, --interactive', 'open prompt to query parameters') .action( async ( language: string | undefined, client: string | undefined, - { verbose } + { verbose, interactive } ) => { - language = await promptLanguage(language); - client = await promptClient(client, CLIENTS_JS); + language = await promptLanguage(language, interactive); + client = await promptClient(client, interactive, CLIENTS_JS); await buildClients( generatorList({ language, client, clientList: CLIENTS_JS }), @@ -162,13 +168,14 @@ buildCommand ]) ) .option('-v, --verbose', 'make the verification verbose') + .option('-i, --interactive', 'open prompt to query parameters') .action( async ( client: string | undefined, outputFormat: 'json' | 'yml' | undefined, - { verbose } + { verbose, interactive } ) => { - client = await promptClient(client); + client = await promptClient(client, interactive); if (!outputFormat) { outputFormat = 'yml'; @@ -178,7 +185,8 @@ buildCommand if (client === 'all') { clientsTodo = CLIENTS; } - await buildSpecs(clientsTodo, outputFormat!, Boolean(verbose), true); + // ignore cache when building from cli + await buildSpecs(clientsTodo, outputFormat!, Boolean(verbose), false); } ); @@ -196,14 +204,15 @@ ctsCommand new Argument('[client]', 'The client').choices(['all'].concat(CLIENTS)) ) .option('-v, --verbose', 'make the generation verbose') + .option('-i, --interactive', 'open prompt to query parameters') .action( async ( language: string | undefined, client: string | undefined, - { verbose } + { verbose, interactive } ) => { - language = await promptLanguage(language); - client = await promptClient(client); + language = await promptLanguage(language, interactive); + client = await promptClient(client, interactive); await ctsGenerateMany( generatorList({ language, client }), @@ -221,8 +230,9 @@ ctsCommand ) ) .option('-v, --verbose', 'make the generation verbose') - .action(async (language: string | undefined, { verbose }) => { - language = await promptLanguage(language); + .option('-i, --interactive', 'open prompt to query parameters') + .action(async (language: string | undefined, { verbose, interactive }) => { + language = await promptLanguage(language, interactive); let langsTodo = [language]; if (language === 'all') { @@ -238,15 +248,22 @@ program .addArgument( new Argument('[client]', 'The client').choices(['all'].concat(CLIENTS)) ) - .action(async (language: string | undefined, client: string | undefined) => { - language = await promptLanguage(language); - client = await promptClient(client); - - await playground({ - language, - client, - key: createGeneratorKey({ language, client }), - }); - }); + .option('-i, --interactive', 'open prompt to query parameters') + .action( + async ( + language: string | undefined, + client: string | undefined, + { interactive } + ) => { + language = await promptLanguage(language, interactive); + client = await promptClient(client, interactive); + + await playground({ + language, + client, + key: createGeneratorKey({ language, client }), + }); + } + ); program.parse(); diff --git a/scripts/release/process-release.ts b/scripts/release/process-release.ts index 3dec99ed08..e95fa565fa 100755 --- a/scripts/release/process-release.ts +++ b/scripts/release/process-release.ts @@ -109,13 +109,13 @@ async function processRelease(): Promise { // generate clients to release for (const lang of Object.keys(versionsToRelease)) { console.log(`Generating ${lang} client(s)...`); - await run(`yarn api generate ${lang} all`); + await run(`yarn cli generate ${lang}`); } // generate clients to just update the repos for (const lang of langsToUpdateRepo) { console.log(`Generating ${lang} client(s)...`); - await run(`yarn api generate ${lang} all`, { verbose: true }); + await run(`yarn cli generate ${lang}`, { verbose: true }); } const clientPath = toAbsolutePath( From fcd80ef44f82ed3cec46e35d651b8e337c69fdfc Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Fri, 25 Feb 2022 13:29:46 +0100 Subject: [PATCH 19/20] eunjae's review --- scripts/buildClients.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/scripts/buildClients.ts b/scripts/buildClients.ts index 5facbf1189..09701f3aa4 100644 --- a/scripts/buildClients.ts +++ b/scripts/buildClients.ts @@ -3,6 +3,11 @@ import { getLanguageFolder } from './config'; import { createSpinner } from './oraLog'; import type { Generator } from './types'; +const multiBuildLanguage = new Set(['javascript']); + +/** + * Build only a specific client for one language, used by javascript for example. + */ async function buildPerClient( { language, key, additionalProperties: { packageName } }: Generator, verbose: boolean @@ -21,6 +26,9 @@ async function buildPerClient( spinner.succeed(); } +/** + * Build all client for a language at the same time, for those who live in the same folder. + */ async function buildAllClients( language: string, verbose: boolean @@ -48,21 +56,17 @@ export async function buildClients( generators: Generator[], verbose: boolean ): Promise { - // For javascript, we build each client individually, - // we should detect this from the config file, for example `isMutliBuild` - - // for other languages, we can mutualize the build const langs = [...new Set(generators.map((gen) => gen.language))]; await Promise.all([ Promise.all( generators - .filter(({ language }) => language === 'javascript') + .filter(({ language }) => multiBuildLanguage.has(language)) .map((gen) => buildPerClient(gen, verbose)) ), Promise.all( langs - .filter((lang) => lang !== 'javascript') + .filter((lang) => !multiBuildLanguage.has(lang)) .map((lang) => buildAllClients(lang, verbose)) ), ]); From 9fbb509de75c3bbfff68e079a2a4449c7835b98d Mon Sep 17 00:00:00 2001 From: Pierre Millot Date: Fri, 25 Feb 2022 15:08:40 +0100 Subject: [PATCH 20/20] quotes --- README.md | 4 ++++ scripts/buildClients.ts | 2 +- scripts/buildSpecs.ts | 18 +++++++----------- scripts/common.ts | 11 +++++++---- scripts/cts/runCts.ts | 7 +++++-- scripts/formatter.ts | 2 +- 6 files changed, 25 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 069cc6044f..5bdcc5c5a1 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,10 @@ yarn docker build clients java recommend You can add `-v` to almost every command to have a more verbose output. +### Interactive command + +If you want to choose the language and client from a list you can add the `--interactive` option, or `-i`. + ## Testing clients You can test our generated clients by running: diff --git a/scripts/buildClients.ts b/scripts/buildClients.ts index 09701f3aa4..082505e841 100644 --- a/scripts/buildClients.ts +++ b/scripts/buildClients.ts @@ -33,7 +33,7 @@ async function buildAllClients( language: string, verbose: boolean ): Promise { - const spinner = createSpinner(`building ${language}`, verbose).start(); + const spinner = createSpinner(`building '${language}'`, verbose).start(); switch (language) { case 'java': await run( diff --git a/scripts/buildSpecs.ts b/scripts/buildSpecs.ts index 1311928dfb..44bf1e6208 100644 --- a/scripts/buildSpecs.ts +++ b/scripts/buildSpecs.ts @@ -11,11 +11,11 @@ async function buildSpec( verbose: boolean, useCache: boolean ): Promise { - createSpinner(`${client} spec`, verbose).start().info(); + createSpinner(`'${client}' spec`, verbose).start().info(); const cacheFile = toAbsolutePath(`specs/dist/${client}.cache`); if (useCache) { const spinner = createSpinner( - `checking cache for ${client}`, + `checking cache for '${client}'`, verbose ).start(); // check if file and cache exist @@ -33,19 +33,19 @@ async function buildSpec( } } - spinner.info(`cache not found for ${client} spec`); + spinner.info(`cache not found for ${client}' spec`); } - const spinner = createSpinner(`linting ${client} spec`, verbose).start(); + const spinner = createSpinner(`linting '${client}' spec`, verbose).start(); await run(`yarn specs:lint ${client}`, { verbose }); - spinner.text = `building ${client} spec`; + spinner.text = `building '${client}' spec`; await run( `yarn openapi bundle specs/${client}/spec.yml -o specs/bundled/${client}.${outputFormat} --ext ${outputFormat}`, { verbose } ); - spinner.text = `validating ${client} spec`; + spinner.text = `validating '${client}' spec`; await run(`yarn openapi lint specs/bundled/${client}.${outputFormat}`, { verbose, }); @@ -54,11 +54,7 @@ async function buildSpec( const hash = (await hashElement(toAbsolutePath(`specs/${client}`))).hash; await fsp.writeFile(cacheFile, hash); - spinner.succeed(); - - createSpinner(`building complete for ${client} spec`, verbose) - .start() - .succeed(); + spinner.succeed(`building complete for '${client}' spec`); } export async function buildSpecs( diff --git a/scripts/common.ts b/scripts/common.ts index 58fb5a3c95..41dcf7643d 100644 --- a/scripts/common.ts +++ b/scripts/common.ts @@ -29,10 +29,13 @@ export const CLIENTS = [ export const CLIENTS_JS = CLIENTS.concat([]); -export function splitGeneratorKey(key: string): Generator { - const language = key.slice(0, key.indexOf('-')); - const client = key.slice(key.indexOf('-') + 1); - return { language, client, key }; +/** + * Takes a generator key in the form 'language-client' and returns the Generator object. + */ +export function splitGeneratorKey(generatorKey: string): Generator { + const language = generatorKey.slice(0, generatorKey.indexOf('-')); + const client = generatorKey.slice(generatorKey.indexOf('-') + 1); + return { language, client, key: generatorKey }; } export function createGeneratorKey({ diff --git a/scripts/cts/runCts.ts b/scripts/cts/runCts.ts index 34bbb0ace9..9b637cc57d 100644 --- a/scripts/cts/runCts.ts +++ b/scripts/cts/runCts.ts @@ -2,7 +2,10 @@ import { run } from '../common'; import { createSpinner } from '../oraLog'; async function runCtsOne(language: string, verbose: boolean): Promise { - const spinner = createSpinner(`running cts for ${language}`, verbose).start(); + const spinner = createSpinner( + `running cts for '${language}'`, + verbose + ).start(); switch (language) { case 'javascript': await run('yarn workspace javascript-tests test', { verbose }); @@ -23,7 +26,7 @@ async function runCtsOne(language: string, verbose: boolean): Promise { break; }*/ default: - spinner.warn(`skipping unknown language ${language} to run the CTS`); + spinner.warn(`skipping unknown language '${language}' to run the CTS`); return; } spinner.succeed(); diff --git a/scripts/formatter.ts b/scripts/formatter.ts index 0c72d9bfc4..b35a5918f5 100644 --- a/scripts/formatter.ts +++ b/scripts/formatter.ts @@ -7,7 +7,7 @@ export async function formatter( verbose = false ): Promise { const spinner = createSpinner( - { text: `formatting ${language}`, indent: 4 }, + { text: `formatting '${language}'`, indent: 4 }, verbose ).start(); let cmd = '';