From edbd459180fcbd10a67e32943601a0672462339b Mon Sep 17 00:00:00 2001 From: Daniel Bluhm Date: Thu, 22 Dec 2022 11:40:18 -0500 Subject: [PATCH] ci: add gha for pr-tests Signed-off-by: Daniel Bluhm --- .github/workflows/pr-tests.yml | 20 +++ .github/workflows/tests-indy.yml | 58 +++++++ .github/workflows/tests.yml | 35 +++++ docker/Dockerfile.indy | 260 +++++++++++++++++++++++++++++++ 4 files changed, 373 insertions(+) create mode 100644 .github/workflows/pr-tests.yml create mode 100644 .github/workflows/tests-indy.yml create mode 100644 .github/workflows/tests.yml create mode 100644 docker/Dockerfile.indy diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml new file mode 100644 index 0000000000..851ea9cf35 --- /dev/null +++ b/.github/workflows/pr-tests.yml @@ -0,0 +1,20 @@ +name: PR Tests + +on: + pull_request: + +jobs: + tests: + name: Tests + uses: ./.github/workflows/tests.yml + with: + python-version: "3.6" + os: "ubuntu-20.04" + + tests-indy: + name: Tests (Indy) + uses: ./.github/workflows/tests-indy.yml + with: + python-version: "3.6" + indy-version: "1.16.0" + os: "ubuntu-20.04" diff --git a/.github/workflows/tests-indy.yml b/.github/workflows/tests-indy.yml new file mode 100644 index 0000000000..a893acf5b5 --- /dev/null +++ b/.github/workflows/tests-indy.yml @@ -0,0 +1,58 @@ +name: Tests (Indy) + +on: + workflow_call: + inputs: + python-version: + required: true + type: string + indy-version: + required: true + type: string + os: + required: true + type: string + +jobs: + tests: + name: Test Python ${{ inputs.python-version }} on Indy ${{ inputs.indy-version }} + runs-on: ${{ inputs.os }} + steps: + - uses: actions/checkout@v3 + + - name: Cache image layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache-test + key: ${{ runner.os }}-buildx-test-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-test- + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Build test image + uses: docker/build-push-action@v3 + with: + load: true + context: . + file: docker/Dockerfile.indy + target: acapy-test + tags: acapy-test:latest + build-args: | + python_version=${{ inputs.python-version }} + indy_version=${{ inputs.indy-version }} + cache-from: type=local,src=/tmp/.buildx-cache-test + cache-to: type=local,dest=/tmp/.buildx-cache-test-new,mode=max + + # Temp fix + # https://github.com/docker/build-push-action/issues/252 + # https://github.com/moby/buildkit/issues/1896 + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache-test + mv /tmp/.buildx-cache-test-new /tmp/.buildx-cache-test + + - name: Run pytest + run: | + docker run --rm acapy-test:latest diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000000..919d1e4f21 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,35 @@ +name: Tests + +on: + workflow_call: + inputs: + python-version: + required: true + type: string + os: + required: true + type: string + +jobs: + tests: + name: Test Python ${{ inputs.python-version }} + runs-on: ${{ inputs.os }} + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ inputs.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python-version }} + cache: 'pip' + cache-dependency-path: 'requirements*.txt' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip3 install --no-cache-dir \ + -r requirements.txt \ + -r requirements.askar.txt \ + -r requirements.bbs.txt \ + -r requirements.dev.txt + - name: Tests + run: | + pytest diff --git a/docker/Dockerfile.indy b/docker/Dockerfile.indy new file mode 100644 index 0000000000..1072998985 --- /dev/null +++ b/docker/Dockerfile.indy @@ -0,0 +1,260 @@ +ARG python_version=3.6.13 +ARG rust_version=1.46 + +# This image could be replaced with an "indy" image from another repo, +# such as the indy-sdk +FROM rust:${rust_version}-slim-buster as indy-builder + +ARG user=indy +ENV HOME="/home/$user" +WORKDIR $HOME +RUN mkdir -p .local/bin .local/etc .local/lib + +# Install environment +RUN apt-get update -y && \ + apt-get install -y --no-install-recommends \ + automake \ + build-essential \ + ca-certificates \ + cmake \ + curl \ + git \ + libbz2-dev \ + libffi-dev \ + libgmp-dev \ + liblzma-dev \ + libncurses5-dev \ + libncursesw5-dev \ + libsecp256k1-dev \ + libsodium-dev \ + libsqlite3-dev \ + libssl-dev \ + libtool \ + libzmq3-dev \ + pkg-config \ + zlib1g-dev && \ + rm -rf /var/lib/apt/lists/* + +# set to --release for smaller, optimized library +ARG indy_build_flags=--release + +ARG indy_version=1.16.0 +ARG indy_sdk_url=https://codeload.github.com/hyperledger/indy-sdk/tar.gz/refs/tags/v${indy_version} + +# make local libs and binaries accessible +ENV PATH="$HOME/.local/bin:$PATH" +ENV LIBRARY_PATH="$HOME/.local/lib:$LIBRARY_PATH" + +# Download and extract indy-sdk +RUN mkdir indy-sdk && \ + curl "${indy_sdk_url}" | tar -xz -C indy-sdk + +# Build and install indy-sdk +WORKDIR $HOME/indy-sdk +RUN cd indy-sdk*/libindy && \ + cargo build ${indy_build_flags} && \ + cp target/*/libindy.so "$HOME/.local/lib" && \ + cargo clean + +# Package python3-indy +RUN tar czvf ../python3-indy.tgz -C indy-sdk*/wrappers/python . + +# grab the latest sdk code for the postgres plug-in +WORKDIR $HOME +ARG indy_postgres_url=${indy_sdk_url} +RUN mkdir indy-postgres && \ + curl "${indy_postgres_url}" | tar -xz -C indy-postgres + +# Build and install postgres_storage plugin +WORKDIR $HOME/indy-postgres +RUN cd indy-sdk*/experimental/plugins/postgres_storage && \ + cargo build ${indy_build_flags} && \ + cp target/*/libindystrgpostgres.so "$HOME/.local/lib" && \ + cargo clean + +# Clean up SDK +WORKDIR $HOME +RUN rm -rf indy-sdk indy-postgres + + +# Indy Base Image +# This image could be replaced with an "indy-python" image from another repo, +# such as the indy-sdk +FROM python:${python_version}-slim-buster as indy-base + +ARG uid=1001 +ARG user=indy +ARG indy_version + +ENV HOME="/home/$user" \ + APP_ROOT="$HOME" \ + LC_ALL=C.UTF-8 \ + LANG=C.UTF-8 \ + PIP_NO_CACHE_DIR=off \ + PYTHONUNBUFFERED=1 \ + PYTHONIOENCODING=UTF-8 \ + RUST_LOG=warning \ + SHELL=/bin/bash \ + SUMMARY="indy-python base image" \ + DESCRIPTION="aries-cloudagent provides a base image for running Hyperledger Aries agents in Docker. \ + This image provides all the necessary dependencies to use the indy-sdk in python. Based on Debian Buster." + +LABEL summary="$SUMMARY" \ + description="$DESCRIPTION" \ + io.k8s.description="$DESCRIPTION" \ + io.k8s.display-name="indy-python $indy_version" \ + name="indy-python" \ + version="$indy_version" \ + maintainer="" + +# Add indy user +RUN useradd -U -ms /bin/bash -u $uid $user + +# Install environment +RUN apt-get update -y && \ + apt-get install -y --no-install-recommends \ + apt-transport-https \ + ca-certificates \ + bzip2 \ + curl \ + git \ + less \ + libffi6 \ + libgmp10 \ + liblzma5 \ + libncurses5 \ + libncursesw5 \ + libsecp256k1-0 \ + libzmq5 \ + net-tools \ + openssl \ + sqlite3 \ + vim-tiny \ + zlib1g && \ + rm -rf /var/lib/apt/lists/* /usr/share/doc/* + +WORKDIR $HOME + +# Copy build results +COPY --from=indy-builder --chown=$user:$user $HOME . + +RUN mkdir -p $HOME/.local/bin + +# Add local binaries and aliases to path +ENV PATH="$HOME/.local/bin:$PATH" + +# Make libraries resolvable by python +ENV LD_LIBRARY_PATH="$HOME/.local/lib:$LD_LIBRARY_PATH" +RUN echo "$HOME/.local/lib" > /etc/ld.so.conf.d/local.conf && ldconfig + +# Install python3-indy +RUN pip install --no-cache-dir python3-indy.tgz && rm python3-indy.tgz + +# - In order to drop the root user, we have to make some directories writable +# to the root group as OpenShift default security model is to run the container +# under random UID. +RUN usermod -a -G 0 $user + +# Create standard directories to allow volume mounting and set permissions +# Note: PIP_NO_CACHE_DIR environment variable should be cleared to allow caching +RUN mkdir -p \ + $HOME/.cache/pip/http \ + $HOME/.indy-cli/networks \ + $HOME/.indy_client/wallet \ + $HOME/.indy_client/pool \ + $HOME/.indy_client/ledger-cache \ + $HOME/ledger/sandbox/data \ + $HOME/log + +# The root group needs access the directories under $HOME/.indy_client for the container to function in OpenShift. +# Also ensure the permissions on the python 'site-packages' folder are set correctly. +RUN chown -R $user:root $HOME/.indy_client \ + && chmod -R ug+rw $HOME/log $HOME/ledger $HOME/.cache $HOME/.indy_client + +USER $user + +CMD ["bash"] + + +# ACA-Py Test +# Used to run ACA-Py unit tests with Indy +FROM indy-base as acapy-test + +USER indy + +RUN mkdir src test-reports + +WORKDIR /home/indy/src + +RUN mkdir -p test-reports && chown -R indy:indy test-reports && chmod -R ug+rw test-reports + +ADD requirements*.txt ./ + +USER root +RUN pip3 install --no-cache-dir \ + -r requirements.txt \ + -r requirements.askar.txt \ + -r requirements.bbs.txt \ + -r requirements.dev.txt + +ADD --chown=indy:root . . +USER indy + +ENTRYPOINT ["/bin/bash", "-c", "pytest \"$@\"", "--"] + +# ACA-Py Builder +# Build ACA-Py wheel using setuptools +FROM python:${python_version}-slim-buster AS acapy-builder + +WORKDIR /src + +ADD . . + +RUN pip install setuptools wheel +RUN python setup.py sdist bdist_wheel + + +# ACA-Py Indy +# Install wheel from builder and commit final image +FROM indy-base AS main + +ARG uid=1001 +ARG user=indy +ARG acapy_version +ARG acapy_reqs=[askar,bbs] + +ENV HOME="/home/$user" \ + APP_ROOT="$HOME" \ + LC_ALL=C.UTF-8 \ + LANG=C.UTF-8 \ + PIP_NO_CACHE_DIR=off \ + PYTHONUNBUFFERED=1 \ + PYTHONIOENCODING=UTF-8 \ + RUST_LOG=warning \ + SHELL=/bin/bash \ + SUMMARY="aries-cloudagent image" \ + DESCRIPTION="aries-cloudagent provides a base image for running Hyperledger Aries agents in Docker. \ + This image layers the python implementation of aries-cloudagent $acapy_version. \ + This image includes indy-sdk and supporting libraries." + +LABEL summary="$SUMMARY" \ + description="$DESCRIPTION" \ + io.k8s.description="$DESCRIPTION" \ + io.k8s.display-name="aries-cloudagent $acapy_version" \ + name="aries-cloudagent" \ + version="$acapy_version" \ + maintainer="" + +# Create standard directories to allow volume mounting and set permissions +# Note: PIP_NO_CACHE_DIR environment variable should be cleared to allow caching +RUN mkdir -p $HOME/.aries_cloudagent + +# The root group needs access the directories under $HOME/.indy_client for the container to function in OpenShift. +# Also ensure the permissions on the python 'site-packages' folder are set correctly. +RUN chmod -R ug+rw $HOME/.aries_cloudagent + +COPY --from=acapy-builder /src/dist/aries_cloudagent*.whl . + +RUN pip install --no-cache-dir --find-links=. aries_cloudagent${acapy_reqs} && rm aries_cloudagent*.whl + +ENTRYPOINT ["aca-py"]