Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore(CI): add docker-prebuilds into monorepo #885

Merged
merged 2 commits into from
Dec 5, 2022
Merged

chore(CI): add docker-prebuilds into monorepo #885

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1e75ecc
chore(CI): add docker-prebuilds to monorepo
Flouse Dec 3, 2022
187f7dc
chore: set hashFiles('crates/**') as the cache key of godwoken component
Flouse Dec 4, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
227 changes: 227 additions & 0 deletions .github/workflows/docker.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,227 @@
name: Docker

on:
push:
branches: [ 'main', 'dev*', 'v1*', '1.*' ]
# Publish semver tags as releases.
tags: [ 'v*.*.*', '1.*' ]

env:
# Use docker.io for Docker Hub if empty
REGISTRY: ${{ github.ref_type != 'tag' && 'ghcr.io/' || '' }}
# github.repository as <account>/<repo>
IMAGE_NAME: godwoken-prebuilds


jobs:
docker-build-push:
runs-on: ubuntu-20.04
# Map the meta step outputs to this job outputs
outputs:
image_name: ${{ steps.result.outputs.image_name }}
image_tag: ${{ steps.result.outputs.image_tag }}
# If you specify the access for any of these scopes, all of those that are not specified are set to none.
permissions:
contents: read
packages: write

steps:
- name: Checkout repository
uses: actions/checkout@v3
with:
submodules: true

- uses: actions/cache@v3
with:
path: |
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
key: ${{ runner.os }}-cargo-${{ github.sha }}
restore-keys: |
${{ runner.os }}-cargo-

- name: Install Rust components
run: rustup component add rustfmt && rustup component add clippy
- name: Install moleculec
run: |
test "$(moleculec --version)" = "Moleculec 0.7.2" \
|| cargo install moleculec --version 0.7.2 --force
- name: Install capsule
env:
CAPSULE_VERSION: v0.7.0
run: |
(which capsule && test "$(capsule --version)" = "Capsule 0.7.0") \
|| curl -OL https://github.com/nervosnetwork/capsule/releases/download/${CAPSULE_VERSION}/capsule_${CAPSULE_VERSION}_x86_64-linux.tar.gz \
&& tar xf capsule_${CAPSULE_VERSION}_x86_64-linux.tar.gz \
&& mv capsule_${CAPSULE_VERSION}_x86_64-linux/capsule ~/.cargo/bin/
capsule --version

# Login against a Docker registry except on PR
# https://github.com/docker/login-action
# GitHub automatically creates a unique GITHUB_TOKEN secret to use in this workflow.
- name: Log into registry ${{ env.REGISTRY }}
if: github.event_name != 'pull_request'
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.ref_type != 'tag' && github.repository_owner || secrets.DOCKERHUB_USERNAME }}
password: ${{ github.ref_type != 'tag' && secrets.GITHUB_TOKEN || secrets.DOCKERHUB_TOKEN }}

- name: Prepare components
id: prepare
working-directory: docker
run: |
make prepare-repos
echo "Record the component's reference to the outputs of this step"
cat build/versions >> $GITHUB_OUTPUT

- name: Print the references of components
run: |
echo ref.component.godwoken=${{ steps.prepare.outputs.GODWOKEN_REF }}
echo ref.component.gwos=${{ steps.prepare.outputs.GODWOKEN_REF }}
echo ref.component.gwos-evm=${{ steps.prepare.outputs.GODWOKEN_REF }}
echo ref.component.ckb-production-scripts=${{ steps.prepare.outputs.OMNI_LOCK_REF }}

- name: Cache of component.ckb-production-scripts
id: ckb-production-scripts-cache
uses: actions/cache@v3
with:
path: docker/build/ckb-production-scripts/build/omni_lock
key: component.ckb-production-scripts-${{ steps.prepare.outputs.ckb-production-scripts-sha1 }}
- name: Build omni_lock
if: steps.ckb-production-scripts-cache.outputs.cache-hit != 'true'
working-directory: docker/build/ckb-production-scripts
run: make all-via-docker

- name: Cache of component.gwos
id: gwos-cache
uses: actions/cache@v3
with:
path: |
gwos/build/release/*
gwos/c/build/*-generator
gwos/c/build/*-validator
gwos/c/build/account_locks/*
key: component.gwos-${{ hashFiles('gwos/**') }}
- name: Build gwos binaries
if: steps.gwos-cache.outputs.cache-hit != 'true'
working-directory: gwos
run: cd c && make && cd .. && capsule build --release --debug-output

- name: Cache of component.gwos-evm
id: godwoken-polyjuice-cache
uses: actions/cache@v3
with:
path: |
gwos-evm/build/*generator*
gwos-evm/build/*validator*
key: component.gwos-evm-${{ hashFiles('gwos-evm/**') }}
- name: Build godwoken-polyjuice
if: steps.godwoken-polyjuice-cache.outputs.cache-hit != 'true'
working-directory: gwos-evm
run: |
git submodule update --init --recursive --depth=1
make all-via-docker

- name: Cache of component.godwoken
id: godwoken-cache
uses: actions/cache@v3
with:
path: |
target/release/godwoken
target/release/gw-tools
key: component.godwoken-crates-${{ hashFiles('crates/**') }}
- name: Cache Godwoken target directory
if: steps.godwoken-cache.outputs.cache-hit != 'true'
uses: actions/cache@v3
with:
path: |
target
key: ${{ runner.os }}-focal-cargo-${{ github.sha }}
restore-keys: |
${{ runner.os }}-focal-cargo-
- name: Build godwoken
if: steps.godwoken-cache.outputs.cache-hit != 'true'
# PORTABLE=1 USE_SSE=1 tell rocksdb to target AVX2, that means "mostly portable".
# see also: https://github.com/nervosnetwork/rust-rocksdb/blob/2de3ae5e5e23a315a477bd24e700eb4f5ef7a378/librocksdb-sys/build_detect_platform#L696-L699
run: |
rustup component add rustfmt
PORTABLE=1 USE_SSE=1 RUSTFLAGS="-C target-cpu=skylake" CARGO_PROFILE_RELEASE_LTO=true cargo build --release

# Extract metadata (tags, labels) for Docker
# https://github.com/docker/metadata-action
- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@v4
with:
images: ${{ env.REGISTRY }}${{ startsWith(github.ref, 'refs/tags') && github.repository_owner == 'godwokenrises' && 'nervos' || github.repository_owner }}/${{ env.IMAGE_NAME }}
# dynamically set date as a suffix
tags: |
type=ref,event=tag
type=ref,event=branch,suffix=-{{date 'YYYYMMDDHHmm'}}
type=ref,event=branch
labels: |
maintainer=Godwoken Core Dev
org.opencontainers.image.authors=Godwoken Core Dev
source.component.godwoken=https://github.com/godwokenrises/godwoken/tree/${{steps.prepare.outputs.GODWOKEN_REF }}
source.component.gwos=https://github.com/godwokenrises/godwoken/tree/${{steps.prepare.outputs.GODWOKEN_REF }}/gwos
source.component.gwos-evm=https://github.com/godwokenrises/godwoken/tree/${{steps.prepare.outputs.GODWOKEN_REF }}/gwos-evm
source.component.ckb-production-scripts=https://github.com/nervosnetwork/ckb-production-scripts/tree/${{steps.prepare.outputs.OMNI_LOCK_REF }}
ref.component.godwoken=${{ steps.prepare.outputs.GODWOKEN_REF }}
ref.component.godwoken-sha1=${{ steps.prepare.outputs.godwoken-sha1 }}
ref.component.gwos=${{ steps.prepare.outputs.GODWOKEN_REF }}
ref.component.gwos-sha1=${{ steps.prepare.outputs.godwoken-sha1 }}
ref.component.gwos-evm=${{ steps.prepare.outputs.GODWOKEN_REF }}
ref.component.gwos-evm-sha1=${{ steps.prepare.outputs.godwoken-sha1 }}
ref.component.ckb-production-scripts=${{ steps.prepare.outputs.OMNI_LOCK_REF }}
ref.component.ckb-production-scripts-sha1=${{ steps.prepare.outputs.ckb-production-scripts-sha1 }}

# Build and push Docker image with Buildx (don't push on PR)
# https://github.com/docker/build-push-action
- name: Build and push Docker image to ${{ env.REGISTRY }}${{ github.repository_owner }}/${{ env.IMAGE_NAME }}
if: ${{ github.ref_type != 'tag' }}
uses: docker/build-push-action@v3
with:
context: .
file: docker/Dockerfile
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}

# Build and push Docker image with Buildx (don't push on PR)
# only for new tag
- name: Build and push Docker image to https://hub.docker.com/r/nervos/godwoken-prebuilds
if: ${{ github.repository_owner == 'godwokenrises' && startsWith(github.ref, 'refs/tags') }}
uses: docker/build-push-action@v3
with:
context: .
file: docker/Dockerfile
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}

- name: Check versions of the binaries in ${{ fromJSON(steps.meta.outputs.json).tags[0] }}
if: ${{ github.event_name != 'pull_request' }}
env:
IMAGE: ${{ fromJSON(steps.meta.outputs.json).tags[0] }}
run: |
docker run --rm ${{ env.IMAGE }} godwoken --version
docker run --rm ${{ env.IMAGE }} gw-tools --version
docker run --rm ${{ env.IMAGE }} ckb --version
docker run --rm ${{ env.IMAGE }} ckb-cli --version
docker run --rm ${{ env.IMAGE }} find /scripts -type f -exec sha1sum {} \;

- name: Record image info to the outputs of this jobs
id: result
run: |
echo "image_name=`echo ${{ fromJSON(steps.meta.outputs.json).tags[0] }} | awk -F ':' '{print $1}'`" >> $GITHUB_OUTPUT
echo "image_tag=`echo ${{ fromJSON(steps.meta.outputs.json).tags[0] }} | awk -F ':' '{print $NF}'`" >> $GITHUB_OUTPUT

integration-test:
needs: docker-build-push
uses: godwokenrises/godwoken-tests/.github/workflows/reusable-integration-test-v1.yml@develop
with:
extra_github_env: |
GODWOKEN_PREBUILD_IMAGE_NAME="${{ needs.docker-build-push.outputs.image_name }}:${{ needs.docker-build-push.outputs.image_tag }}"
80 changes: 80 additions & 0 deletions docker/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
# Historical versions refer to checksum.txt
# https://github.com/godwokenrises/godwoken-docker-prebuilds/pkgs/container/godwoken-prebuilds/49615211?tag=dev-poly1.5.0-202211091419
FROM ghcr.io/godwokenrises/godwoken-prebuilds:dev-poly1.5.0 as historical-versions

################################################################################

# https://hub.docker.com/_/ubuntu/
FROM ubuntu:focal
LABEL description="Docker image containing all binaries used by Godwoken, saving you the hassles of building them yourself."
LABEL maintainer="Godwoken Core Dev"

RUN mkdir -p /scripts/godwoken-scripts \
&& mkdir -p /scripts/godwoken-polyjuice \
&& mkdir /ckb

RUN apt-get update \
&& apt-get dist-upgrade -y \
&& apt-get install -y curl jq \
&& apt-get clean \
&& echo 'Finished installing OS updates'

# ckb
RUN cd /ckb \
&& curl -LO https://github.com/nervosnetwork/ckb/releases/download/v0.103.0/ckb_v0.103.0_x86_64-unknown-linux-gnu.tar.gz \
&& tar xzf ckb_v0.103.0_x86_64-unknown-linux-gnu.tar.gz \
&& cp ckb_v0.103.0_x86_64-unknown-linux-gnu/ckb /bin/ckb \
&& cp ckb_v0.103.0_x86_64-unknown-linux-gnu/ckb-cli /bin/ckb-cli \
&& rm -rf /ckb

# Copy historical versions (refer to checksum.txt)
#
# If <dest> doesn’t exist, it is created along with all missing directories in its path.
# refer to https://docs.docker.com/engine/reference/builder/#copy
COPY docker/checksum.txt /scripts/
COPY --from=historical-versions /scripts/godwoken-polyjuice-v1.1.5-beta/ \
/scripts/godwoken-polyjuice-v1.1.5-beta/
COPY --from=historical-versions /scripts/godwoken-polyjuice-v1.2.0/ \
/scripts/godwoken-polyjuice-v1.2.0/
COPY --from=historical-versions /scripts/godwoken-polyjuice-v1.4.0/ \
/scripts/godwoken-polyjuice-v1.4.0/
COPY --from=historical-versions /scripts/godwoken-polyjuice-v1.4.1/ \
/scripts/godwoken-polyjuice-v1.4.1/
COPY --from=historical-versions /scripts/godwoken-polyjuice-v1.4.2/ \
/scripts/godwoken-polyjuice-v1.4.2/
COPY --from=historical-versions /scripts/godwoken-polyjuice-v1.4.4/ \
/scripts/godwoken-polyjuice-v1.4.4/
COPY --from=historical-versions /scripts/godwoken-polyjuice-v1.4.5/ \
/scripts/godwoken-polyjuice-v1.4.5/
COPY --from=historical-versions /scripts/godwoken-polyjuice/* \
/scripts/godwoken-polyjuice-v1.5.0/

#################################### latest ####################################
# COPY [--chown=<user>:<group>] ["<src>",... "<dest>"]

# /scripts/godwoken-polyjuice
COPY gwos-evm/build/*generator* \
gwos-evm/build/*validator* \
/scripts/godwoken-polyjuice/
# TODO: remove *.aot in Polyjuice Makefile
# RUN find /scripts -type f -name '*.aot' -exec rm {} \;

# /scripts/omni-lock and /scripts/godwoken-scripts
COPY docker/build/ckb-production-scripts/build/omni_lock \
gwos/build/release/* \
gwos/c/build/*-generator \
gwos/c/build/*-validator \
gwos/c/build/account_locks/* \
/scripts/godwoken-scripts/

# godwoken
COPY target/release/godwoken \
target/release/gw-tools \
docker/gw-healthcheck.sh \
/bin/
################################################################################


WORKDIR /deploy

CMD [ "godwoken", "--version" ]
64 changes: 64 additions & 0 deletions docker/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
SHELL := /bin/bash

# components repos
OMNI_LOCK_REPO := https://github.com/nervosnetwork/ckb-production-scripts.git

# components tags
OMNI_LOCK_REF := rc_lock

define prepare_repo
if [ ! -d "build/$(3)" ]; then\
git clone --depth=1 $(1) build/$(3);\
fi
cd build/$(3);\
git fetch origin $(2);\
git checkout FETCH_HEAD;\
git submodule update --init --recursive --depth=1;\
echo "$(3)-sha1=$$(git rev-parse HEAD)" >> ../versions
endef

prepare-repos:
mkdir -p build
echo "godwoken-sha1=$$(git rev-parse HEAD)" >> build/versions
echo "GODWOKEN_REF=$$(git describe --tags --exact-match 2> /dev/null || git symbolic-ref -q --short HEAD || git rev-parse --short HEAD) $$(git rev-parse --short HEAD)" >> build/versions
$(call prepare_repo,$(OMNI_LOCK_REPO),$(OMNI_LOCK_REF),ckb-production-scripts)
echo "OMNI_LOCK_REF=$(OMNI_LOCK_REF) $$(cd build/ckb-production-scripts && git rev-parse --short HEAD)" >> build/versions

build-components: prepare-repos
cd build/ckb-production-scripts && make all-via-docker
cd ../gwos-evm && git submodule update --init --recursive --depth=1 && make dist && cd -
cd ../gwos && cd c && make && cd .. && capsule build --release --debug-output && cd ../..
cd ../ && rustup component add rustfmt && RUSTFLAGS="-C target-cpu=native" CARGO_PROFILE_RELEASE_LTO=true cargo build --release

build-push:
make build-components
@read -p "Please Enter New Image Tag: " VERSION ; \
docker build . -t nervos/godwoken-prebuilds:$$VERSION ; \
docker push nervos/godwoken-prebuilds:$$VERSION

test:
make build-components
docker build . -t godwokenrises/godwoken-prebuilds:latest-test
mkdir -p `pwd`/test-result/scripts
mkdir -p `pwd`/test-result/bin
docker run -it -d --name dummy godwokenrises/godwoken-prebuilds:latest-test
docker cp dummy:/scripts/. `pwd`/test-result/scripts
docker cp dummy:/bin/godwoken `pwd`/test-result/bin
docker cp dummy:/bin/gw-tools `pwd`/test-result/bin
docker rm -f dummy
make test-files

test-files:
echo "start checking build result..."
# compare scripts files
make test-scripts-files
make test-polyjuice-files
# compare bin files
cd `pwd`/test-result/bin && ./godwoken --version && ./gw-tools --version
[ -e "test-result" ] && rm -rf test-result

test-scripts-files:
source tool.sh && check_scripts_files_exists

test-polyjuice-files:
source tool.sh && check_polyjuice_files_exists
Loading