From 2fb32d745d342b8d6755b840689a0085f08cc6fd Mon Sep 17 00:00:00 2001 From: Michal Kimle Date: Tue, 11 Oct 2022 13:16:49 +0200 Subject: [PATCH] Enable building Docker images for released packages Also cleanup and modularization of packaging scripts. --- .changeset/config.json | 2 +- .dockerignore | 1 + .github/workflows/build-test.yml | 68 ++++++----- .gitignore | 2 + docker/Dockerfile | 12 +- docker/README.md | 130 ++++++++++++++++---- docker/entrypoint.sh | 21 ---- docker/scripts/build-docker-images.ts | 28 +++++ docker/scripts/cli.ts | 133 +++++++++++++++++++++ docker/scripts/npm-registry.ts | 63 ++++++++++ docker/scripts/prepare-containers.ts | 99 --------------- docker/scripts/publish-packages.ts | 56 +++++++-- docker/scripts/utils.ts | 4 +- package.json | 21 +++- packages/airnode-admin/docker/README.md | 2 +- packages/airnode-deployer/docker/README.md | 2 +- packages/airnode-node/docker/README.md | 2 +- yarn.lock | 5 + 18 files changed, 450 insertions(+), 201 deletions(-) delete mode 100755 docker/entrypoint.sh create mode 100644 docker/scripts/build-docker-images.ts create mode 100644 docker/scripts/cli.ts create mode 100644 docker/scripts/npm-registry.ts delete mode 100644 docker/scripts/prepare-containers.ts diff --git a/.changeset/config.json b/.changeset/config.json index 0acca6a68c..ecb3075a62 100644 --- a/.changeset/config.json +++ b/.changeset/config.json @@ -1,6 +1,6 @@ { "$schema": "https://unpkg.com/@changesets/config@1.6.1/schema.json", - "changelog": "@changesets/cli/changelog", + "changelog": ["@changesets/changelog-github", { "repo": "api3dao/airnode" }], "commit": false, "fixed": [["@api3/*"]], "access": "public", diff --git a/.dockerignore b/.dockerignore index 2df714cec1..a68958f310 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,6 +2,7 @@ .github/ **/node_modules/** **/dist/** +!docker/scripts/dist **/build/** **/coverage/** diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index f1f9d1b0ef..4e31da1617 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -67,36 +67,6 @@ jobs: with: path: ./* key: pre-build-${{ github.sha }} - docker-build: - name: Build all the Docker containers and push them to Docker Hub - runs-on: ubuntu-latest - # Don't run twice for a push within an internal PR - if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository - needs: pre-build - steps: - - uses: actions/cache@v3 - id: cache-pre-build - with: - path: ./* - key: pre-build-${{ github.sha }} - - name: Setup Node - uses: actions/setup-node@v3 - with: - node-version: ${{ env.TARGET_NODE_VERSION }} - cache: yarn - - name: Build Docker containers - run: yarn docker:build --dev --npm-tag ${{ github.sha }} --docker-tag ${{ github.sha }} - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Push airnode-admin to Docker Hub - run: docker push api3/airnode-admin-dev:${{ github.sha }} - - name: Push airnode-client to Docker Hub - run: docker push api3/airnode-client-dev:${{ github.sha }} - - name: Push airnode-deployer to Docker Hub - run: docker push api3/airnode-deployer-dev:${{ github.sha }} build: name: Build and lint Airnode runs-on: ubuntu-latest @@ -135,6 +105,44 @@ jobs: with: path: ./* key: ${{ github.sha }} + docker-build: + name: Build all the Docker containers and push them to Docker Hub + runs-on: ubuntu-latest + # Don't run twice for a push within an internal PR + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + needs: build + steps: + - uses: actions/cache@v3 + id: restore-build + with: + path: ./* + key: ${{ github.sha }} + - name: Setup Node + uses: actions/setup-node@v3 + with: + node-version: ${{ env.TARGET_NODE_VERSION }} + cache: yarn + - name: Build Docker packaging container + run: yarn docker:build:packaging + - name: Start local NPM registry + run: yarn docker:scripts:npm-registry:start + - name: Publish NPM packages to local NPM registry + run: yarn docker:scripts:publish-packages --npm-registry local --npm-tag ${{ github.sha }} --snapshot + - name: Build Docker containers + run: yarn docker:scripts:build-docker-images --dev --npm-registry local --npm-tag ${{ github.sha }} --docker-tag ${{ github.sha }} + - name: Stop local NPM registry + run: yarn docker:scripts:npm-registry:stop + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Push airnode-admin to Docker Hub + run: docker push api3/airnode-admin-dev:${{ github.sha }} + - name: Push airnode-client to Docker Hub + run: docker push api3/airnode-client-dev:${{ github.sha }} + - name: Push airnode-deployer to Docker Hub + run: docker push api3/airnode-deployer-dev:${{ github.sha }} unit-tests: name: Unit tests runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 32ea7d998b..8b420bb684 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,5 @@ package.json_ # Other .DS_Store + +docker/scripts/dist diff --git a/docker/Dockerfile b/docker/Dockerfile index e4abb545f9..e7a059c05b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ FROM node:14.19.3-alpine3.15 AS environment -ENV buildDir="/build" +ENV appDir="/app" RUN apk add --update --no-cache git rsync docker $([ $(arch) == "aarch64" ] && echo "python3 make g++") && \ yarn global add npm && \ @@ -12,8 +12,12 @@ RUN apk add --update --no-cache git rsync docker $([ $(arch) == "aarch64" ] && e wget -O /root/.cache/hardhat-nodejs/compilers/linux-amd64/solc-linux-amd64-v0.8.9+commit.e5eed63a https://binaries.soliditylang.org/linux-amd64/solc-linux-amd64-v0.8.9+commit.e5eed63a && \ wget -O /root/.cache/hardhat-nodejs/compilers/linux-amd64/list.json https://solc-bin.ethereum.org/linux-amd64/list.json -COPY ./entrypoint.sh /entrypoint.sh +COPY docker/scripts/dist/*.js ${appDir}/ +COPY docker/scripts/dist/*.map ${appDir}/ +COPY packages/airnode-admin/docker ${appDir}/airnode-admin +COPY packages/airnode-deployer/docker ${appDir}/airnode-deployer +COPY packages/airnode-node/docker ${appDir}/airnode-client -WORKDIR ${buildDir} +WORKDIR ${appDir} -ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file +ENTRYPOINT ["node", "/app/index.js"] diff --git a/docker/README.md b/docker/README.md index 73eaaa11ff..99b7930440 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,16 +1,13 @@ # api3/airnode-packaging -This is a Docker container for building all the packages in the monorepo, publishing the NPM packages (TODO) and -building the Docker containers. +This is a Docker container that can: -## How it works +- Start/stop a local NPM registry Docker container +- Build and publish NPM packages to both local and official (TODO) NPM registry +- Build Docker containers from both local and official NPM packages The container uses so called Docker-in-Docker method to build packages and Docker container in the clean Dockerized -environment. The building flow looks like this: - -1. The local NPM registry is spun up -2. Packages are built and published in the local NPM registry -3. Docker containers are built, installing the packages from the local NPM registry +environment. ## Build @@ -26,41 +23,124 @@ allow conditional build steps. You can read more about how to enable it in its ## Usage -In order for the container to work correctly you need to pass the Docker daemon socket from your system, so the -containers can be used "within" the containers: +There are three CLI commands available: + +- [`npm-registry`](#npm-registry) +- [`publish-packages`](#publish-packages) +- [`build-docker-images`](#build-docker-images) + +**To run all the pieces together and build Docker images, you can use the two convenience Yarn targets:** ```bash -... -v /var/run/docker.sock:/var/run/docker.sock ... +yarn docker:build:local +yarn docker:build:latest +``` + +### npm-registry + +``` +Manages the local NPM registry + +Commands: + index.js npm-registry start Start the local NPM registry + index.js npm-registry stop Stop the local NPM registry + +Options: + --version Show version number [boolean] + --help Show help [boolean] ``` -If you mount your Airnode directory, the content will be used to build the packages & containers (the content is copied, -there are no changes to your local files): +You can start and stop a local NPM registry. Can be useful for manual package testing but it's mostly a step needed for +the rest of the functionality. + +Example: ```bash -... -v $(pwd):/airnode ... +docker run -it --rm -v /var/run/docker.sock:/var/run/docker.sock api3/airnode-packaging:latest npm-registry start ``` -If there's no mount to `/airnode`, the content will be retrieved from the GitHub repository. By setting the `GIT_REF` -variable you can specify a Git reference (e.g. branch, commit hash, ...) that should be used: +You can use two convenience Yarn targets for starting and stopping the container: ```bash -... -e GIT_REF=my-dev-branch ... +yarn docker:scripts:npm-registry:start +yarn docker:scripts:npm-registry:stop +``` + +### publish-packages + ``` +Publish NPM packages + +Options: + --version Show version number [boolean] + --help Show help [boolean] + -r, --npm-registry NPM registry URL to publish to or a keyword `local` to use a local NPM registry + [string] [default: "https://registry.npmjs.org/"] + -t, --npm-tag NPM tag to publish the packages under [string] [default: "latest"] + -s, --snapshot Publish in a snapshot mode + (https://github.com/changesets/changesets/blob/main/docs/snapshot-releases.md) + [boolean] [default: false] +``` + +You can build and publish NPM packages. + +Use the `--npm-registry` option to specify the registry where the packages should be uploaded to. When the keyword +`local` is used instead of the URL, the local NPM (see [`npm-registry`](#npm-registry)) will be used. + +Use the `--npm-tag` option to specify the tag for the published packages. -There's a simple CLI via which you can change some build attributes: +Use the `--snapshot` option to publish a +[snapshot package](https://github.com/changesets/changesets/blob/main/docs/snapshot-releases.md) -- `--dev` - builds the dev Docker images with the `-dev` suffix used in the CI (default: `false`) -- `--npm-tag` - string used as the NPM tag (default: `local`) -- `--docker-tag` - string used as the Docker tag (default: `local`) +Example: ```bash -... api3/airnode-packaging:latest --dev --npm-tag 9c218e333a4c27103e64b5b13b1fb53abbcd56c5 --docker-tag 9c218e333a4c27103e64b5b13b1fb53abbcd56c5 +docker run --rm -v $(pwd):/airnode -v /var/run/docker.sock:/var/run/docker.sock api3/airnode-packaging:latest publish-packages --npm-registry local --npm-tag local --snapshot ``` -The whole run command may look something like: +You can use a convenience Yarn target to publish snapshot packages to a local NPM registry: ```bash -docker run -it --rm -v $(pwd):/airnode -v /var/run/docker.sock:/var/run/docker.sock api3/airnode-packaging:latest --dev --npm-tag 9c218e333a4c27103e64b5b13b1fb53abbcd56c5 --docker-tag 9c218e333a4c27103e64b5b13b1fb53abbcd56c5 +yarn docker:scripts:publish-packages:local ``` -You can also use the `build:docker:images` target to build containers from your Airnode directory tagged as `local`. +### build-docker-images + +``` +Build Docker images + +Options: + --version Show version number [boolean] + --help Show help [boolean] + -r, --npm-registry NPM registry URL to fetch packages from or a keyword `local` to use a local NPM registry + [string] [default: "https://registry.npmjs.org/"] + -t, --npm-tag NPM tag/version of the packages that will be fetched [string] [default: "latest"] + -g, --docker-tag Docker tag to build the images under [string] [default: "latest"] + -d, --dev Build Docker dev images (with -dev suffix) [boolean] [default: false] +``` + +You can build Airnode Docker images. + +Use the `--npm-registry` option to specify the registry from which the NPM packages should be installed during the +building process. When the keyword `local` is used instead of the URL, the local NPM (see +[`npm-registry`](#npm-registry)) will be used. + +Use the `--npm-tag` option to specify the tag of the NPM package that should be installed during the building process. + +Use the `--docker-tag` option to specify the Docker tag the resulting images will have. + +Use the `--dev` option to build the development images, with the `-dev` suffix in their name. + +Example: + +```bash +docker run --rm -v /var/run/docker.sock:/var/run/docker.sock api3/airnode-packaging:latest build-docker-images --npm-registry local --npm-tag local --docker-tag local +``` + +You can use two convenience Yarn targets for building Docker images from the local NPM packages and from the latest +official ones: + +```bash +yarn docker:scripts:build-docker-images:local +yarn docker:scripts:build-docker-images:latest +``` diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh deleted file mode 100755 index c22f9e21d4..0000000000 --- a/docker/entrypoint.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/sh - -set -e - -GIT_REF=${GIT_REF:-master} - -# Use Git respository as a source when there's no `/airnode` directory mounted -if [ -d /airnode ]; then - # Efficient way to copy the whole directory with a lot of small files - # Keeping any local code changes but excluding platform-specific files (dependencies, builds, etc.) - git config --global --add safe.directory /airnode - rsync -a --include .git --exclude-from=<(git -C /airnode ls-files --exclude-standard -oi --directory) /airnode/ /build -else - git clone https://github.com/api3dao/airnode.git . - git checkout ${GIT_REF} -fi - -yarn bootstrap -yarn build - -yarn ts-node docker/scripts/prepare-containers.ts "$@" diff --git a/docker/scripts/build-docker-images.ts b/docker/scripts/build-docker-images.ts new file mode 100644 index 0000000000..a6c4d9b4ba --- /dev/null +++ b/docker/scripts/build-docker-images.ts @@ -0,0 +1,28 @@ +import { getNpmRegistryContainer } from './npm-registry'; +import { runCommand, unifyUrlFormat } from './utils'; + +export const buildDockerImages = (npmRegistry: string, npmTag: string, dockerTag: string, dev: boolean) => { + let npmRegistryUrl = npmRegistry; + const devSuffix = dev ? '-dev' : ''; + + if (npmRegistry === 'local') { + const npmRegistryInfo = getNpmRegistryContainer(); + if (!npmRegistryInfo) { + throw new Error(`Can't build Docker images: No local NPM registry running`); + } + + npmRegistryUrl = npmRegistryInfo.npmRegistryUrl; + } + + npmRegistryUrl = unifyUrlFormat(npmRegistryUrl); + + runCommand( + `docker build --no-cache --build-arg npmRegistryUrl=${npmRegistryUrl} --build-arg npmTag=${npmTag} --tag api3/airnode-admin${devSuffix}:${dockerTag} --file /app/airnode-admin/Dockerfile /app/airnode-admin` + ); + runCommand( + `docker build --no-cache --build-arg npmRegistryUrl=${npmRegistryUrl} --build-arg npmTag=${npmTag} --tag api3/airnode-deployer${devSuffix}:${dockerTag} --file /app/airnode-deployer/Dockerfile /app/airnode-deployer` + ); + runCommand( + `docker build --no-cache --build-arg npmRegistryUrl=${npmRegistryUrl} --build-arg npmTag=${npmTag} --tag api3/airnode-client${devSuffix}:${dockerTag} --file /app/airnode-client/Dockerfile /app/airnode-client` + ); +}; diff --git a/docker/scripts/cli.ts b/docker/scripts/cli.ts new file mode 100644 index 0000000000..e504d6ce04 --- /dev/null +++ b/docker/scripts/cli.ts @@ -0,0 +1,133 @@ +import yargs from 'yargs'; +import omitBy from 'lodash/omitBy'; +import { logger } from '@api3/airnode-utilities'; +import { go, GoResult, goSync } from '@api3/promise-utils'; +import { stopNpmRegistry, startNpmRegistry } from './npm-registry'; +import { buildDockerImages } from './build-docker-images'; +import { publishPackages } from './publish-packages'; + +// Taken from airnode-deployer +const longArguments = (args: Record) => { + return JSON.stringify(omitBy(args, (_, arg) => arg === '$0' || arg.length === 1)); +}; + +const handleCliCommand = (result: GoResult) => { + if (!result.success) { + logger.error(result.error.message); + // eslint-disable-next-line functional/immutable-data + process.exitCode = 1; + } +}; + +const runAsyncCliCommand = async (command: () => Promise) => { + const goCommand = await go(command); + handleCliCommand(goCommand); +}; + +const runCliCommand = (command: () => void) => { + const goCommand = goSync(command); + handleCliCommand(goCommand); +}; + +yargs(process.argv.slice(2)) + .command('npm-registry', 'Manages the local NPM registry', (yargs) => { + yargs + .command('start', 'Start the local NPM registry', {}, (args) => { + logger.log(`Running command '${args._[0]} ${args._[1]}' with arguments ${longArguments(args)}`); + runAsyncCliCommand(async () => { + const npmRegistryInfo = await startNpmRegistry(); + logger.log(''); + logger.log(`NPM registry container name: ${npmRegistryInfo.npmRegistryContainerName}`); + logger.log(`NPM registry URL: ${npmRegistryInfo.npmRegistryUrl}`); + }); + }) + .command('stop', 'Stop the local NPM registry', {}, (args) => { + logger.log(`Running command '${args._[0]} ${args._[1]}' with arguments ${longArguments(args)}`); + runCliCommand(() => { + stopNpmRegistry(); + }); + }) + .help() + .demandCommand(1) + .strict(); + }) + .command( + 'publish-packages', + 'Publish NPM packages', + { + 'npm-registry': { + alias: 'r', + description: 'NPM registry URL to publish to or a keyword `local` to use a local NPM registry', + default: 'https://registry.npmjs.org/', + type: 'string', + }, + 'npm-tag': { + alias: 't', + description: 'NPM tag to publish the packages under', + default: 'latest', + type: 'string', + }, + snapshot: { + alias: 's', + description: + 'Publish in a snapshot mode (https://github.com/changesets/changesets/blob/main/docs/snapshot-releases.md)', + default: false, + type: 'boolean', + }, + }, + (args) => { + logger.log(`Running command '${args._[0]}' with arguments ${longArguments(args)}`); + + // Temporary check for not yet supported functionality + if (args.npmRegistry === 'https://registry.npmjs.org/') { + throw new Error('Publishing packages to the official NPM registry is not supported yet'); + } + if (!args.snapshot) { + throw new Error('Only snapshot packages are supported at the moment'); + } + + runCliCommand(() => { + publishPackages(args.npmRegistry, args.npmTag, args.snapshot); + }); + } + ) + .command( + 'build-docker-images', + 'Build Docker images', + { + 'npm-registry': { + alias: 'r', + description: 'NPM registry URL to fetch packages from or a keyword `local` to use a local NPM registry', + default: 'https://registry.npmjs.org/', + type: 'string', + }, + 'npm-tag': { + alias: 't', + description: 'NPM tag/version of the packages that will be fetched', + default: 'latest', + type: 'string', + }, + 'docker-tag': { + alias: 'g', + description: 'Docker tag to build the images under', + default: 'latest', + type: 'string', + }, + dev: { + alias: 'd', + description: 'Build Docker dev images (with -dev suffix)', + default: false, + type: 'boolean', + }, + }, + (args) => { + logger.log(`Running command '${args._[0]}' with arguments ${longArguments(args)}`); + runCliCommand(() => { + buildDockerImages(args.npmRegistry, args.npmTag, args.dockerTag, args.dev); + }); + } + ) + .help() + .demandCommand(1) + .strict() + .wrap(120).argv; diff --git a/docker/scripts/npm-registry.ts b/docker/scripts/npm-registry.ts new file mode 100644 index 0000000000..54cc8c6b3a --- /dev/null +++ b/docker/scripts/npm-registry.ts @@ -0,0 +1,63 @@ +import { randomBytes } from 'crypto'; +import axios from 'axios'; +import { go } from '@api3/promise-utils'; +import { logger } from '@api3/airnode-utilities'; +import { runCommand } from './utils'; + +const NPM_REGISTRY_CONTAINER_NAME_PREFIX = 'airnode-npm-registry'; +const NPM_REGISTRY_CONTAINER_IMAGE = 'verdaccio/verdaccio:5.15'; +const NPM_REGISTRY_CONTIANER_PORT = 4873; + +export const getNpmRegistryContainer = () => { + const containerNames = runCommand(`docker ps --format '{{.Names}}'`).split('\n'); + const npmRegistryContainerName = containerNames.filter((name) => + name.startsWith(NPM_REGISTRY_CONTAINER_NAME_PREFIX) + )[0]; + + if (!npmRegistryContainerName) return null; + + const npmRegistryContainerIp = runCommand( + `docker inspect --format '{{.NetworkSettings.IPAddress}}' ${npmRegistryContainerName}` + ); + const npmRegistryUrl = `http://${npmRegistryContainerIp}:${NPM_REGISTRY_CONTIANER_PORT}`; + + return { + npmRegistryContainerName, + npmRegistryUrl, + }; +}; + +export const stopNpmRegistry = () => { + const npmRegistryInfo = getNpmRegistryContainer(); + if (!npmRegistryInfo) { + throw new Error('No local NPM registry found'); + } + + runCommand(`docker stop ${npmRegistryInfo.npmRegistryContainerName}`); +}; + +export const startNpmRegistry = async () => { + const npmRegistryContainerName = `airnode-npm-registry-${randomBytes(4).toString('hex')}`; + runCommand(`docker run -d --rm --name ${npmRegistryContainerName} ${NPM_REGISTRY_CONTAINER_IMAGE}`); + + const npmRegistryInfo = getNpmRegistryContainer(); + if (!npmRegistryInfo) { + throw new Error('Starting a local NPM registry failed'); + } + + logger.log( + `Running NPM registry container. Name: ${npmRegistryInfo.npmRegistryContainerName}, URL: ${npmRegistryInfo.npmRegistryUrl}` + ); + + // Waiting for container to become ready + const goPing = await go(() => axios.get(npmRegistryInfo.npmRegistryUrl), { + retries: 10, + delay: { type: 'static', delayMs: 1500 }, + }); + if (!goPing.success) { + stopNpmRegistry(); + throw new Error(`Can't connect to the local NPM registry: ${goPing.error}`); + } + + return npmRegistryInfo; +}; diff --git a/docker/scripts/prepare-containers.ts b/docker/scripts/prepare-containers.ts deleted file mode 100644 index 5c4503ffe0..0000000000 --- a/docker/scripts/prepare-containers.ts +++ /dev/null @@ -1,99 +0,0 @@ -///////////////////////////////////////// WARNING ///////////////////////////////////////// -// // -// Do NOT run these scripts outside a Docker container! // -// They modify the environment they run in and therefore may change your local settings. // -// // -///////////////////////////////////////// WARNING ///////////////////////////////////////// - -import { randomBytes } from 'crypto'; -import axios from 'axios'; -import yargs from 'yargs'; -import { go } from '@api3/promise-utils'; -import { logger } from '@api3/airnode-utilities'; -import { runCommand, isDocker } from './utils'; -import { publishPackages } from './publish-packages'; - -const NPM_REGISTRY_CONTAINER_IMAGE = 'verdaccio/verdaccio:latest'; -const NPM_REGISTRY_CONTIANER_PORT = 4873; - -if (!isDocker()) { - throw new Error('This script should be run only in the Docker container'); -} - -const runNpmRegistry = async () => { - const npmRegistryContainerName = `airnode-npm-registry-${randomBytes(4).toString('hex')}`; - runCommand(`docker run -d --rm --name ${npmRegistryContainerName} ${NPM_REGISTRY_CONTAINER_IMAGE}`); - - const npmRegistryContainerIp = runCommand( - `docker inspect --format '{{.NetworkSettings.IPAddress}}' ${npmRegistryContainerName}` - ) - .toString() - .trim(); - const npmRegistryContainerUrl = `http://${npmRegistryContainerIp}:${NPM_REGISTRY_CONTIANER_PORT}`; - - logger.log(`Running NPM registry container. Name: ${npmRegistryContainerName}, URL: ${npmRegistryContainerUrl}`); - - // Waiting for container to become ready - const goPing = await go(() => axios.get(npmRegistryContainerUrl), { - retries: 10, - delay: { type: 'static', delayMs: 1000 }, - }); - if (!goPing.success) { - stopNpmRegistry(npmRegistryContainerName); - throw new Error(`Can't connect to the local NPM registry: ${goPing.error}`); - } - - return { - npmRegistryContainerName, - npmRegistryContainerUrl, - }; -}; - -const stopNpmRegistry = (npmRegistryContainerName: string) => runCommand(`docker stop ${npmRegistryContainerName}`); - -const buildContainers = (npmRegistryContainerUrl: string, npmTag: string, dockerTag: string, dev: boolean) => { - const devSuffix = dev ? '-dev' : ''; - - runCommand( - `docker build --no-cache --build-arg npmRegistryUrl=${npmRegistryContainerUrl} --build-arg npmTag=${npmTag} --tag api3/airnode-admin${devSuffix}:${dockerTag} --file packages/airnode-admin/docker/Dockerfile packages/airnode-admin/docker` - ); - runCommand( - `docker build --no-cache --build-arg npmRegistryUrl=${npmRegistryContainerUrl} --build-arg npmTag=${npmTag} --tag api3/airnode-deployer${devSuffix}:${dockerTag} --file packages/airnode-deployer/docker/Dockerfile packages/airnode-deployer/docker` - ); - runCommand( - `docker build --no-cache --build-arg npmRegistryUrl=${npmRegistryContainerUrl} --build-arg npmTag=${npmTag} --tag api3/airnode-client${devSuffix}:${dockerTag} --file packages/airnode-node/docker/Dockerfile packages/airnode-node/docker` - ); -}; - -const prepareContainers = async (npmTag: string, dockerTag: string, dev: boolean) => { - const { npmRegistryContainerName, npmRegistryContainerUrl } = await runNpmRegistry(); - await publishPackages(npmRegistryContainerUrl, npmTag); - buildContainers(npmRegistryContainerUrl, npmTag, dockerTag, dev); - stopNpmRegistry(npmRegistryContainerName); -}; - -// TODO: CLI will be extended to allow package publication to the official NPM registry and without building a Docker containers -// https://github.com/api3dao/airnode/issues/1332 -// https://github.com/api3dao/airnode/issues/1333 -const options = yargs(process.argv.slice(2)) - .option('npmTag', { - default: 'local', - type: 'string', - description: 'NPM tag', - }) - .option('dockerTag', { - default: 'local', - type: 'string', - description: 'Docker tag', - }) - .option('dev', { - default: false, - type: 'boolean', - description: 'Create development Docker images', - }) - .strict() - .wrap(120) - .parseSync(); -logger.log(`CLI options: ${JSON.stringify(options)}`); - -prepareContainers(options.npmTag, options.dockerTag, options.dev); diff --git a/docker/scripts/publish-packages.ts b/docker/scripts/publish-packages.ts index 799058d72e..b5983a16bf 100755 --- a/docker/scripts/publish-packages.ts +++ b/docker/scripts/publish-packages.ts @@ -5,21 +5,42 @@ // // ///////////////////////////////////////// WARNING ///////////////////////////////////////// -import { readFileSync, writeFileSync } from 'fs'; +import { readFileSync, writeFileSync, accessSync, constants } from 'fs'; import { homedir } from 'os'; import { join } from 'path'; import { randomBytes } from 'crypto'; import axios from 'axios'; -import { go } from '@api3/promise-utils'; +import { go, goSync } from '@api3/promise-utils'; import { logger } from '@api3/airnode-utilities'; -import { runCommand, isDocker } from './utils'; +import { runCommand, isDocker, unifyUrlFormat } from './utils'; +import { getNpmRegistryContainer } from './npm-registry'; -const DEFAULT_NPM_REGISTRY = 'https://registry.npmjs.org'; +const AIRNODE_REPOSITORY = 'https://github.com/api3dao/airnode.git'; if (!isDocker()) { throw new Error('This script should be run only in the Docker container'); } +const fetchProject = () => { + const goAccess = goSync(() => accessSync('/airnode', constants.F_OK)); + if (goAccess.success) { + runCommand('git config --global --add safe.directory /airnode'); + const excludedFiles = runCommand('git -C /airnode ls-files --exclude-standard -oi --directory').split('\n'); + const excludeOptions = excludedFiles.map((excludedFile) => `--exclude ${excludedFile}`).join(' '); + runCommand(`rsync -a ${excludeOptions} --exclude .git /airnode/ /build`); + } else { + const gitRef = process.env.GIT_REF ?? 'master'; + runCommand(`git clone ${AIRNODE_REPOSITORY} /build`); + runCommand(`git -C /build checkout ${gitRef}`); + } +}; + +const buildProject = () => { + runCommand('git config --global --add safe.directory /build'); + runCommand('yarn bootstrap', { cwd: '/build' }); + runCommand('yarn build', { cwd: '/build' }); +}; + const authNpmRegistry = async (npmRegistryUrl: string) => { const dummyUser = randomBytes(4).toString('hex'); const dummyPassword = randomBytes(4).toString('hex'); @@ -66,7 +87,7 @@ registry=${npmRegistryUrl} // This is annoying especially when we want to publish packages only for testing purposes. // By changing the type to `@changesets/cli/changelog` we can circumvent the issue. const simplifyChangesetConfig = () => { - const changesetConfigPath = join(__dirname, '..', '..', '.changeset', 'config.json'); + const changesetConfigPath = '/build/.changeset/config.json'; const oldChangesetConfig = readFileSync(changesetConfigPath, 'utf-8'); const newChangesetConfig = oldChangesetConfig.replace( '"changelog": ["@changesets/changelog-github", { "repo": "api3dao/airnode" }],', @@ -75,16 +96,25 @@ const simplifyChangesetConfig = () => { writeFileSync(changesetConfigPath, newChangesetConfig); }; -export const publishPackages = async (npmRegistryUrl: string, npmTag: string) => { - // Unify the URL format - npmRegistryUrl = npmRegistryUrl.endsWith('/') ? npmRegistryUrl.slice(0, -1) : npmRegistryUrl; - // Publishing packages to the official NPM registry is not supported yet. - if (npmRegistryUrl === DEFAULT_NPM_REGISTRY) - throw new Error('Publishing packages to the official NPM registry is not supported yet.'); +export const publishPackages = async (npmRegistry: string, npmTag: string, _snapshot: boolean) => { + let npmRegistryUrl = npmRegistry; + + if (npmRegistry === 'local') { + const npmRegistryInfo = getNpmRegistryContainer(); + if (!npmRegistryInfo) { + throw new Error(`Can't publish NPM packages: No local NPM registry running`); + } + + npmRegistryUrl = npmRegistryInfo.npmRegistryUrl; + } + + npmRegistryUrl = unifyUrlFormat(npmRegistryUrl); + fetchProject(); + buildProject(); await authNpmRegistry(npmRegistryUrl); simplifyChangesetConfig(); // Ignoring commands' outputs because of the weird text colorization. - runCommand(`yarn changeset version --snapshot ${npmTag}`, { stdio: 'ignore' }); - runCommand(`yarn changeset publish --no-git-tag --snapshot --tag ${npmTag}`, { stdio: 'ignore' }); + runCommand(`yarn changeset version --snapshot ${npmTag}`, { cwd: '/build', stdio: 'ignore' }); + runCommand(`yarn changeset publish --no-git-tag --snapshot --tag ${npmTag}`, { cwd: '/build', stdio: 'ignore' }); }; diff --git a/docker/scripts/utils.ts b/docker/scripts/utils.ts index 82643432a0..898de626e3 100644 --- a/docker/scripts/utils.ts +++ b/docker/scripts/utils.ts @@ -4,9 +4,11 @@ import { logger } from '@api3/airnode-utilities'; export const runCommand = (command: string, options?: ExecSyncOptions) => { logger.log(`Running command: '${command}'${options ? ` with options ${JSON.stringify(options)}` : ''}`); - return execSync(command, options); + return execSync(command, options)?.toString().trim(); }; +export const unifyUrlFormat = (url: string) => (url.endsWith('/') ? url.slice(0, -1) : url); + // Taken from https://github.com/sindresorhus/is-docker let isDockerCached: boolean | undefined; diff --git a/package.json b/package.json index 5c71397b55..70e14deb3e 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "changeset:check": "changeset status --since=origin/master", "changeset:empty": "changeset --empty", "changeset:new-version": "yarn changeset version && yarn ts-node scripts/update-config-fixtures.ts", - "clean": "lerna run clean --stream", + "clean": "rimraf -rf docker/scripts/dist && lerna run clean --stream", "cli:deployer": "lerna run --scope @api3/airnode-deployer cli -- --", "compile": "yarn tsc --build", "dev:api": "(cd packages/airnode-operation && yarn run dev:api)", @@ -45,9 +45,20 @@ "dev:invoke": "(cd packages/airnode-node && yarn run dev:invoke)", "dev:list": "(cd packages/airnode-operation && yarn run dev:list)", "dev:stop": "(cd packages/airnode-operation && yarn run dev:stop)", - "docker:build": "yarn docker:build:packaging && yarn docker:build:images", - "docker:build:images": "docker run --rm -v $(pwd):/airnode -v /var/run/docker.sock:/var/run/docker.sock api3/airnode-packaging:latest", - "docker:build:packaging": "docker build --tag api3/airnode-packaging:latest --file docker/Dockerfile docker/", + "docker:build:local": "yarn docker:build:packaging && yarn docker:build:images:local", + "docker:build:latest": "yarn docker:build:packaging && yarn docker:build:images:latest", + "docker:build:images:latest": "yarn docker:scripts:build-docker-images:latest", + "docker:build:images:local": "yarn docker:scripts:npm-registry:start && yarn docker:scripts:publish-packages:local && yarn docker:scripts:build-docker-images:local && yarn docker:scripts:npm-registry:stop", + "docker:build:packaging": "yarn docker:scripts:build && docker build --tag api3/airnode-packaging:latest --file docker/Dockerfile .", + "docker:scripts:build": "ncc build docker/scripts/cli.ts -o docker/scripts/dist --no-cache --minify --source-map --transpile-only", + "docker:scripts:build-docker-images": "docker run --rm -v /var/run/docker.sock:/var/run/docker.sock api3/airnode-packaging:latest build-docker-images", + "docker:scripts:build-docker-images:latest": "yarn docker:scripts:build-docker-images", + "docker:scripts:build-docker-images:local": "yarn docker:scripts:build-docker-images --npm-registry local --npm-tag local --docker-tag local", + "docker:scripts:npm-registry": "docker run --rm -v /var/run/docker.sock:/var/run/docker.sock api3/airnode-packaging:latest npm-registry", + "docker:scripts:npm-registry:start": "yarn docker:scripts:npm-registry start", + "docker:scripts:npm-registry:stop": "yarn docker:scripts:npm-registry stop", + "docker:scripts:publish-packages": "docker run --rm -v $(pwd):/airnode -v /var/run/docker.sock:/var/run/docker.sock api3/airnode-packaging:latest publish-packages", + "docker:scripts:publish-packages:local": "yarn docker:scripts:publish-packages --npm-registry local --npm-tag local --snapshot", "format:check": "yarn prettier:check && yarn terraform:fmt:check", "format:write": "yarn prettier:write && yarn terraform:fmt:write", "lint": "yarn run lint:eslint && yarn run lint:solhint && yarn format:check && yarn ts-node scripts/validate-ts-references.ts", @@ -91,6 +102,7 @@ "@types/node": "^17.0.18", "@typescript-eslint/eslint-plugin": "^5.38.1", "@typescript-eslint/parser": "^5.38.1", + "@vercel/ncc": "^0.34.0", "axios": "^0.27.2", "comment-json": "^4.2.3", "eslint": "^8.24.0", @@ -102,6 +114,7 @@ "lerna": "^5.5.2", "prettier": "^2.7.1", "prettier-plugin-solidity": "^1.0.0-dev.23", + "rimraf": "^3.0.2", "solhint": "^3.3.7", "ts-node": "^10.9.1", "typescript": "^4.8.3", diff --git a/packages/airnode-admin/docker/README.md b/packages/airnode-admin/docker/README.md index 8a23187e4d..6137e286e8 100644 --- a/packages/airnode-admin/docker/README.md +++ b/packages/airnode-admin/docker/README.md @@ -11,5 +11,5 @@ more about the admin CLI, please read [its documentation](../README.md). In order to build Airnode admin CLI Docker image run the following command from the root directory: ```bash -yarn docker:build +yarn docker:build:local ``` diff --git a/packages/airnode-deployer/docker/README.md b/packages/airnode-deployer/docker/README.md index beab6447c9..b6f26da5e3 100644 --- a/packages/airnode-deployer/docker/README.md +++ b/packages/airnode-deployer/docker/README.md @@ -11,7 +11,7 @@ more about the Deployer, please read [its documentation](../README.md). In order to build Airnode deployer Docker image run the following command from the root directory: ```bash -yarn docker:build +yarn docker:build:local ``` > If building on windows ensure that the `entrypoint.sh` file uses `LF` line endings. Otherwise the image will not be diff --git a/packages/airnode-node/docker/README.md b/packages/airnode-node/docker/README.md index 73026a1342..e57c008822 100644 --- a/packages/airnode-node/docker/README.md +++ b/packages/airnode-node/docker/README.md @@ -11,7 +11,7 @@ please read [its documentation](../README.md). In order to build Airnode Docker image run the following command from the root directory: ```bash -yarn docker:build +yarn docker:build:local ``` > If building on windows ensure that the `airnode-crontab` and `entrypoint.sh` files use `LF` line endings. Otherwise diff --git a/yarn.lock b/yarn.lock index bd45e24991..8888416fab 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3310,6 +3310,11 @@ resolved "https://registry.yarnpkg.com/@ungap/promise-all-settled/-/promise-all-settled-1.1.2.tgz#aa58042711d6e3275dd37dc597e5d31e8c290a44" integrity sha512-sL/cEvJWAnClXw0wHk85/2L0G6Sj8UB0Ctc1TEMbKSsmpRosqhwj9gWgFRZSrBr2f9tiXISwNhCPmlfqUqyb9Q== +"@vercel/ncc@^0.34.0": + version "0.34.0" + resolved "https://registry.yarnpkg.com/@vercel/ncc/-/ncc-0.34.0.tgz#d0139528320e46670d949c82967044a8f66db054" + integrity sha512-G9h5ZLBJ/V57Ou9vz5hI8pda/YQX5HQszCs3AmIus3XzsmRn/0Ptic5otD3xVST8QLKk7AMk7AqpsyQGN7MZ9A== + "@webassemblyjs/ast@1.11.1": version "1.11.1" resolved "https://registry.yarnpkg.com/@webassemblyjs/ast/-/ast-1.11.1.tgz#2bfd767eae1a6996f432ff7e8d7fc75679c0b6a7"