From 0730c6947196825e8cb4c078b5375207202d950a Mon Sep 17 00:00:00 2001 From: Wade Barnes Date: Tue, 6 Jul 2021 10:31:16 -0700 Subject: [PATCH] Add support for publishing Debian packages - Following the pattern and enhancements developed for the plenum workflows. - Update Sovrin signing keys. - Update test runner (`runner.py`) to automatically report the test results for each set of tests. - Enhancements from https://github.com/hyperledger/indy-plenum/commit/06d9406e611784b7005829ca325fcaa3ae435ea5 - Add an option for suppressing the automated junitxml results. - Add `--nojunitxml` flag to allow junitxml results to be suppressed. - Suppress junitxml output when running the Jenkins Pipelines - Generation of junitxml output for the `indy_node/test/auth_rule` tests causes the tests to crash and fail when run via Jenkins. - The GHA workflows use the junitxml results for reporting, and are not affected by the same issue seen with Jenkins. - Address feedback from @udosson Signed-off-by: Wade Barnes --- .gitattributes | 5 + .github/actions/publish-deb/action.yaml | 23 ++ .github/actions/publish-deb/publishPackages | 33 ++ .github/actions/publish-deb/upload-spec.json | 8 + .github/actions/set-version/action.yaml | 63 ++++ .github/workflows/README.md | 28 +- .github/workflows/build.yaml | 348 +++++++++++++++--- .github/workflows/build/Dockerfile | 20 +- .github/workflows/lint/Dockerfile | 2 +- Jenkinsfile.ci | 3 +- .../ubuntu-1604/build-3rd-parties.sh | 6 + ci/ubuntu.dockerfile | 3 + indy_node/__metadata__.py | 2 +- runner.py | 18 +- 14 files changed, 489 insertions(+), 73 deletions(-) create mode 100644 .gitattributes create mode 100644 .github/actions/publish-deb/action.yaml create mode 100755 .github/actions/publish-deb/publishPackages create mode 100644 .github/actions/publish-deb/upload-spec.json create mode 100644 .github/actions/set-version/action.yaml diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..92ce082e5 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +# Set the default behavior, in case people don't have core.autocrlf set. +* text=auto + +# Declare files that will always have LF line endings on checkout. +**/publishPackages text eol=lf \ No newline at end of file diff --git a/.github/actions/publish-deb/action.yaml b/.github/actions/publish-deb/action.yaml new file mode 100644 index 000000000..6b8c56d74 --- /dev/null +++ b/.github/actions/publish-deb/action.yaml @@ -0,0 +1,23 @@ +name: "Publish Debian Packages" +description: > + Publish debian packages to artifactory using the jfrog cli. + This action wraps a script that performs organized publishing of debian packages. + +inputs: + sourceDirectory: + description: "The directory containing the packages to be published." + required: true + distribution: + description: "The distribution supported by the package. i.e xenial, focal, etc." + required: true + component: + description: "The component type. i.e. stable, rc, dev, etc." + required: true + +runs: + using: "composite" + steps: + - name: Publish + shell: bash + run: | + ${{ github.action_path }}/publishPackages "${{ inputs.sourceDirectory }}" "${{ inputs.distribution }}" "${{ inputs.component }}" diff --git a/.github/actions/publish-deb/publishPackages b/.github/actions/publish-deb/publishPackages new file mode 100755 index 000000000..449d7e31a --- /dev/null +++ b/.github/actions/publish-deb/publishPackages @@ -0,0 +1,33 @@ +#!/bin/bash +SCRIPT_HOME="$( cd "$( dirname "$0" )" && pwd )" + +sourceDirectory=${1} +distribution=${2} +component=${3} +uploadSpec=${UPLOAD_SPEC:-${4:-"${SCRIPT_HOME}/upload-spec.json"}} + +export CI=true + +fileList=$(find "${sourceDirectory}" -type f -name '*.deb' -mindepth 1 -maxdepth 1 -printf "%f\n" 2>/dev/null) +for name in ${fileList}; do + shortName=$(echo "${name}" | sed -rn 's~^(python3-)?([^.]*)_.*.deb~\2~p') + architecture=$(echo "${name}" | sed -rn 's~^.*_(.*).deb~\1~p') + startingLetter=$(echo "${shortName}" | sed -rn 's~^(.).*~\1~p') + + echo "=====================================================================================================================" + echo "name: ${name}" + echo "shortName: ${shortName}" + echo "architecture: ${architecture}" + echo "startingLetter: ${startingLetter}" + echo "distribution: ${distribution}" + echo "component: ${component}" + echo "uploadSpec: ${uploadSpec}" + echo "---------------------------------------------------------------------------------------------------------------------" + jfrog rt u \ + --deb ${distribution}/${component}/${architecture} \ + --spec ${uploadSpec}\ + --spec-vars "SOURCE_DIR=${sourceDirectory};PACKAGE_NAME=${name};COMPONENT=${component};PACKAGE_STARTING_LETTER=${startingLetter};PACKAGE_SHORT_NAME=${shortName}" \ + --detailed-summary + echo "=====================================================================================================================" + echo +done \ No newline at end of file diff --git a/.github/actions/publish-deb/upload-spec.json b/.github/actions/publish-deb/upload-spec.json new file mode 100644 index 000000000..e5db11c58 --- /dev/null +++ b/.github/actions/publish-deb/upload-spec.json @@ -0,0 +1,8 @@ +{ + "files": [ + { + "pattern": "${SOURCE_DIR}/${PACKAGE_NAME}", + "target": "indy/pool/${COMPONENT}/${PACKAGE_STARTING_LETTER}/${PACKAGE_SHORT_NAME}/" + } + ] +} \ No newline at end of file diff --git a/.github/actions/set-version/action.yaml b/.github/actions/set-version/action.yaml new file mode 100644 index 000000000..ceef38bdd --- /dev/null +++ b/.github/actions/set-version/action.yaml @@ -0,0 +1,63 @@ +name: "Set Version" +description: "Sets version parameters and makes them available as outputs for subsequent processes." + +inputs: + moduleName: + description: "The name of the module containing the version management APIs for the project; i.e. load_version and set_version" + required: true + default: "indy_node" + isDev: + description: "A flag indicating whether or not this is a dev build; set to either 'true' or 'false'." + required: true + default: false + isRC: + description: "A flag indicating whether or not this is a release candidate build; set to either 'true' or 'false'." + required: true + default: false + +outputs: + targetReleaseVer: + description: "The target release version." + value: ${{ steps.versions.outputs.targetReleaseVer }} + upstreamVer: + description: "The version number to use with Python packages." + value: ${{ steps.versions.outputs.upstreamVer }} + pkgVer: + description: "The version number to use with Debian packages." + value: ${{ steps.versions.outputs.pkgVer }} + +runs: + using: "composite" + steps: + - name: Set Versions + id: versions + shell: bash + run: | + fullVersion=$(python3 -c "from ${{ inputs.moduleName }} import load_version; print(load_version().full)") + release=$(python3 -c "from ${{ inputs.moduleName }} import load_version; print(load_version().release)") + pre=$(python3 -c "from ${{ inputs.moduleName }} import load_version; pre = load_version().parts[-2]; print('' if pre is None else pre)") + revision=$(python3 -c "from ${{ inputs.moduleName }} import load_version; rev = load_version().parts[-1]; print('' if rev is None else rev)") + + targetReleaseVer=${release} + upstreamVer=${targetReleaseVer} + pkgVer=${upstreamVer} + + if [ ${{ inputs.isDev }} == "true" ] || [ ${{ inputs.isRC }} == "true" ]; then + if [ ${{ inputs.isDev }} == "true" ]; then + rev=${{ github.run_number }} + else + rev=${revision} + fi + + upstreamVer+=".${pre}${rev}" + pkgVer+="~${pre}${rev}" + fi + + # Set Outputs ... + echo "::set-output name=targetReleaseVer::${targetReleaseVer}" + echo "::set-output name=upstreamVer::${upstreamVer}" + echo "::set-output name=pkgVer::${pkgVer}" + + echo "Target Release version: ${targetReleaseVer}" + echo "PyPI Release version: ${upstreamVer}" + echo "Debian Release version: ${pkgVer}" \ No newline at end of file diff --git a/.github/workflows/README.md b/.github/workflows/README.md index f1587ac39..3dc4d3e85 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -1,25 +1,29 @@ -### Github Actions Workflow +# GitHub Actions Workflow -This build file replaces the existing `Jenkins.ci` build process. +The workflow in the [build.yaml](build.yaml) file replaces the existing [Jenkins.ci](../../Jenkinsfile.ci) build process. -`lint.yaml` replaces the `Static code validation` stage of the Jenkins build. +The `lint` job replaces the `Static code validation` stage of the Jenkins pipeline, while the remainder of the jobs replace the `Build / Test` stage. -`build.yaml` replaces the `Build / Test` stage of the Jenkins build. +The `Build result notification` stage was not moved to GHA, as build failures will be reports via GHA. Many of the other stages are replaced merely by the fact we're using Github Actions, we use prebuild Docker containers so we don't have to replicate the steps for building containers. -The `Build result notification` stage was not moved to GHA, build failures will be reports via GHA. - The build process for `Jenkins.nightly` was not ported to GHA. -#### Configuring actions +Support for Windows continues as a `ToDo` item. + + +## Configuring actions If you are cloning or forking this repo you will need to configure two secrets for Actions to run correctly. -Secrets can be set via Settings -> Secrets -> New repository secret. +Secrets can be set via Settings -> Secrets -> New repository secret: + +`CR_USER`: is your GH username. It must be lowercase. +`CR_PAT`: can be created by following the [Creating a personal access token](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token) documentation. + +When you create your token, the only permission you need to select is `write:packages` **Upload packages to GitHub package registry**, all other necessary permissions will be selected by default. -CR_USER is your GH username. -CR_PAT can be created by following [these directions](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token) +You may also need to enable [Improved container support](https://docs.github.com/en/packages/guides/enabling-improved-container-support) in order to allow the images to be written to your repository. You'll see an error to this affect if this is the case. -Once you have run the build once with those secrets, you have to make then package public. -Access the package at https://ghcr.io/USER/indy-node/indy-node-build or https://ghcr.io/USER/indy-node/indy-node-lint then change the visibility in 'Package Settings' to 'Public' then re-run the build. +Once you have run the build once with those secrets, you have to make the images public. Access the packages at https://ghcr.io/USER/indy-node/node-build and https://ghcr.io/USER/indy-node/node-lint and change the visibility in 'Package Settings' to 'Public' then re-run the build. Alternatively, if you would prefer to keep the images private, you can manage access to the package and select only the user account associated with the token you setup above. \ No newline at end of file diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 86c37a17d..69017edc7 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -3,116 +3,368 @@ on: [ push, pull_request, workflow_dispatch ] jobs: workflow-setup: + name: Initialize Workflow runs-on: ubuntu-latest outputs: - CACHE_KEY_LINT: ${{ steps.cache.outputs.CACHE_KEY_LINT }} CACHE_KEY_BUILD: ${{ steps.cache.outputs.CACHE_KEY_BUILD }} + CACHE_KEY_LINT: ${{ steps.cache.outputs.CACHE_KEY_LINT }} # Expose the lowercase version of the GitHub repository name # to all subsequent jobs that reference image repositories - # as the push and pull operations require the URL of the repository + # as the push and pull operations require the URL of the repository # to be in lowercase. - GITHUB_REPOSITORY_NAME: ${{ steps.cache.outputs.GITHUB_REPOSITORY_NAME }} + GITHUB_REPOSITORY_NAME: ${{ steps.repository-name.outputs.lowercase }} + GITHUB_REF: ${{ steps.cache.outputs.GITHUB_REF }} + isDev: ${{ steps.build-flags.outputs.isDev }} + isRC: ${{ steps.build-flags.outputs.isRC }} + publish: ${{ steps.build-flags.outputs.publish }} steps: - name: Git checkout uses: actions/checkout@v2 + + - name: Convert the GitHub repository name to lowercase + id: repository-name + uses: ASzc/change-string-case-action@v1 + with: + string: ${{ github.repository }} + - name: Set outputs id: cache run: | - echo "::set-output name=CACHE_KEY_LINT::${{ hashFiles('.github/workflows/lint/Dockerfile') }}" echo "::set-output name=CACHE_KEY_BUILD::${{ hashFiles('.github/workflows/build/Dockerfile') }}" - echo "::set-output name=GITHUB_REPOSITORY_NAME::$(echo ${GITHUB_REPOSITORY,,})" - - build-lint-image: + echo "::set-output name=CACHE_KEY_LINT::${{ hashFiles('.github/workflows/lint/Dockerfile') }}" + + if [[ "${{github.base_ref}}" == 'master' || "${{github.ref}}" == 'refs/heads/master' || "${{github.base_ref}}" == 'main' || "${{github.ref}}" == 'refs/heads/main' ]]; then + echo "::set-output name=GITHUB_REF::main" + elif [[ "${{github.base_ref}}" == 'release*' || "${{github.ref}}" == 'refs/heads/release*' ]]; then + echo "::set-output name=GITHUB_REF::rc" + elif [[ "${{github.base_ref}}" == 'stable' || "${{github.ref}}" == 'refs/heads/stable' ]]; then + echo "::set-output name=GITHUB_REF::stable" + else + echo "::set-output name=GITHUB_REF::dev" + fi + + - name: Set build flags + id: build-flags + run: | + + if [[ "${{steps.cache.outputs.GITHUB_REF}}" == 'dev' || "${{steps.cache.outputs.GITHUB_REF}}" == 'main' ]]; then + echo "::set-output name=isDev::true" + else + echo "::set-output name=isDev::false" + fi + + if [[ "${{steps.cache.outputs.GITHUB_REF}}" == 'rc' ]]; then + echo "::set-output name=isRC::true" + else + echo "::set-output name=isRC::false" + fi + + # Ensure publishing is only performed when the build is executed from the main (hyperledger/indy-node) repository. + if [[ ${{github.event.repository.full_name}} == 'hyperledger/indy-node' && ${{github.event_name}} == 'push' && ( ${{steps.cache.outputs.GITHUB_REF}} == 'main' || ${{steps.cache.outputs.GITHUB_REF}} == 'rc' || ${{steps.cache.outputs.GITHUB_REF}} == 'stable' ) ]]; then + echo "::set-output name=publish::true" + else + echo "::set-output name=publish::false" + fi + + build-image: + name: Create Builder Image + # Reference to workflow-setup job is required to access its various outputs. needs: workflow-setup runs-on: ubuntu-latest env: - DOCKER_BUILDKIT: 1 - CACHE_KEY_LINT: ${{ needs.workflow-setup.outputs.CACHE_KEY_LINT }} + CACHE_KEY_BUILD: ${{ needs.workflow-setup.outputs.CACHE_KEY_BUILD }} GITHUB_REPOSITORY_NAME: ${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }} steps: - name: Git checkout uses: actions/checkout@v2 - - name: Try load from cache. - id: cache-image-lint + + - name: Try load from cache + id: cache-image uses: actions/cache@v2 with: path: ${GITHUB_WORKSPACE}/cache - key: ${{ env.CACHE_KEY_LINT }} - - name: If NOT found in cache, build and push image. - if: steps.cache-image-lint.outputs.cache-hit != 'true' + key: ${{ env.CACHE_KEY_BUILD}} + + - name: Prepare image labels and tags + if: steps.cache-image.outputs.cache-hit != 'true' + id: prep + shell: bash + run: | + DOCKER_IMAGE=ghcr.io/${{ env.GITHUB_REPOSITORY_NAME }}/node-build + # ToDo - Update hard coded 'ubuntu-16-04' tag when integrating these flows with the ubuntu-20.04-upgrade branch. + TAGS="${DOCKER_IMAGE}:latest,${DOCKER_IMAGE}:ubuntu-16-04" + echo ::set-output name=tags::${TAGS} + echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ') + + - name: Log into the GitHub Container Registry + if: steps.cache-image.outputs.cache-hit != 'true' + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + if: steps.cache-image.outputs.cache-hit != 'true' + uses: docker/setup-buildx-action@v1 + + - name: Build and push image + if: steps.cache-image.outputs.cache-hit != 'true' + uses: docker/build-push-action@v2 + with: + context: . + file: .github/workflows/build/Dockerfile + no-cache: true + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.prep.outputs.tags }} + labels: | + org.opencontainers.image.source=${{ github.event.repository.html_url }} + org.opencontainers.image.created=${{ steps.prep.outputs.created }} + org.opencontainers.image.revision=${{ github.sha }} + + - name: Touch Cache + if: steps.cache-image.outputs.cache-hit != 'true' run: | - echo ${{ secrets.CR_PAT }} | docker login ghcr.io --username ${{ secrets.CR_USER }} --password-stdin - docker build -f .github/workflows/lint/Dockerfile --no-cache -t ${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-lint:${{ env.CACHE_KEY_LINT }} . - docker tag ${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-lint:${{ env.CACHE_KEY_LINT }} ghcr.io/${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-lint:latest - docker push ghcr.io/${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-lint:latest mkdir -p ${GITHUB_WORKSPACE}/cache - touch ${GITHUB_WORKSPACE}/cache/${{ env.CACHE_KEY_LINT }} + touch ${GITHUB_WORKSPACE}/cache/${{ env.CACHE_KEY_BUILD }} - build-test-image: + lint-image: + name: Create Linter Image + # Reference to workflow-setup job is required to access its various outputs. needs: workflow-setup runs-on: ubuntu-latest env: - DOCKER_BUILDKIT: 1 - CACHE_KEY_BUILD: ${{ needs.workflow-setup.outputs.CACHE_KEY_BUILD }} + CACHE_KEY_LINT: ${{ needs.workflow-setup.outputs.CACHE_KEY_LINT }} GITHUB_REPOSITORY_NAME: ${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }} steps: - name: Git checkout uses: actions/checkout@v2 - - name: Try load from cache. - id: cache-image-build + + - name: Try load from cache + id: cache-image uses: actions/cache@v2 with: path: ${GITHUB_WORKSPACE}/cache - key: ${{ env.CACHE_KEY_BUILD }} - - name: If NOT found in cache, build and push image. - if: steps.cache-image-build.outputs.cache-hit != 'true' + key: ${{ env.CACHE_KEY_LINT }} + + - name: Prepare image labels and tags + if: steps.cache-image.outputs.cache-hit != 'true' + id: prep + shell: bash + run: | + DOCKER_IMAGE=ghcr.io/${{ env.GITHUB_REPOSITORY_NAME }}/node-lint + # ToDo - Update hard coded 'ubuntu-18-04' tag when integrating these flows with the ubuntu-20.04-upgrade branch. + TAGS="${DOCKER_IMAGE}:latest,${DOCKER_IMAGE}:ubuntu-18-04" + echo ::set-output name=tags::${TAGS} + echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ') + + - name: Log into the GitHub Container Registry + if: steps.cache-image.outputs.cache-hit != 'true' + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + if: steps.cache-image.outputs.cache-hit != 'true' + uses: docker/setup-buildx-action@v1 + + - name: Build and push image + if: steps.cache-image.outputs.cache-hit != 'true' + uses: docker/build-push-action@v2 + with: + context: . + file: .github/workflows/lint/Dockerfile + no-cache: true + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.prep.outputs.tags }} + labels: | + org.opencontainers.image.source=${{ github.event.repository.html_url }} + org.opencontainers.image.created=${{ steps.prep.outputs.created }} + org.opencontainers.image.revision=${{ github.sha }} + + - name: Touch Cache + if: steps.cache-image.outputs.cache-hit != 'true' run: | - echo ${{ secrets.CR_PAT }} | docker login ghcr.io --username ${{ secrets.CR_USER }} --password-stdin - docker build -f .github/workflows/build/Dockerfile --no-cache -t ${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-build:${{ env.CACHE_KEY_BUILD }} . - docker tag ${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-build:${{ env.CACHE_KEY_BUILD }} ghcr.io/${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-build:latest - docker push ghcr.io/${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-build:latest mkdir -p ${GITHUB_WORKSPACE}/cache - touch ${GITHUB_WORKSPACE}/cache/${{ env.CACHE_KEY_BUILD }} + touch ${GITHUB_WORKSPACE}/cache/${{ env.CACHE_KEY_LINT }} - indy_node: - name: Build Indy Node - needs: build-test-image - runs-on: ubuntu-18.04 + indy_node_tests: + name: Sliced Module Tests + # Reference to workflow-setup job is required to access the GITHUB_REPOSITORY_NAME output. + needs: [workflow-setup, build-image] + runs-on: ubuntu-20.04 + # Fix for scacap/action-surefire-report out of memory error: + # - https://github.com/ScaCap/action-surefire-report/issues/17 + env: + NODE_OPTIONS: '--max_old_space_size=4096' container: - image: ghcr.io/${{ github.repository }}/indy-node-build + image: ghcr.io/${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}/node-build:ubuntu-16-04 strategy: matrix: module: [indy_node, indy_common] - slice: [1, 2, 3, 4 ,5, 6, 7,8, 9, 10, 11] + slice: [1, 2, 3, 4 ,5, 6, 7, 8, 9, 10, 11] fail-fast: false steps: - name: Check out code uses: actions/checkout@v2 + # =============================================== + # Caching cannot be used. + # - For some reason as soon as it is enabled + # the tests start complaining about zmq missing + # for the plenum install. + # ----------------------------------------------- + # - name: Cache pip + # uses: actions/cache@v2 + # with: + # # pip cache on the node-build image is not in the default location. + # # path: ~/.cache/pip + # path: /root/.cache/pip + # key: ${{ runner.os }}-indy-node-pip-${{ hashFiles('**/requirements.txt', '**/setup.py') }} + # restore-keys: | + # ${{ runner.os }}-indy-node-pip- + - name: Install dependencies - run: pip install .[tests] - continue-on-error: true + run: | + # Explicitly use the existing pip cache location in the node-build image. + pip --cache-dir /root/.cache/pip install .[tests] - name: Run Indy Node ${{ matrix.module }} test slice ${{ matrix.slice }}/${{ strategy.job-total }} - run: RUSTPYTHONASYNCIODEBUG=0 python3 runner.py --pytest "python3 -m pytest -l -vv --junitxml=test-result-indy-node-${{ matrix.module }}-${{ matrix.slice }}.xml" --dir "${{ matrix.module }}" --output "test-result-indy-node-${{ matrix.slice }}.txt" --test-only-slice "${{ matrix.slice }}/${{ strategy.job-total }}" + id: node-test + run: RUSTPYTHONASYNCIODEBUG=0 python3 runner.py --pytest "python3 -m pytest -l -vv" --dir "${{ matrix.module }}" --output "test-result-node-${{ matrix.slice }}.txt" --test-only-slice "${{ matrix.slice }}/${{ strategy.job-total }}" - name: Publish Test Report - uses: scacap/action-surefire-report@v1 + if: success() || failure() + uses: scacap/action-surefire-report@v1.0.7 continue-on-error: true with: - check_name: Indy Node ${{ matrix.module }} ${{ matrix.slice }}/${{ strategy.job-total }} Test Report + check_name: Indy Node ${{ matrix.module }} Test Report for slice ${{ matrix.slice }}/${{ strategy.job-total }} github_token: ${{ secrets.GITHUB_TOKEN }} - report_paths: test-result-indy-node-${{ matrix.module }}-${{ matrix.slice }}.xml + report_paths: "*-test-results.xml" + + - name: Upload Detailed Test Failure Results + # The test runner only emits the detailed test results if the tests fail. + if: (steps.node-test.outcome == 'failure') && failure() + uses: actions/upload-artifact@v2 + with: + name: detailed-test-result-slice-${{ matrix.slice }} + path: test-result-node-${{ matrix.slice }}.txt + retention-days: 5 lint: name: Lint - runs-on: ubuntu-latest + # Reference to workflow-setup job is required to access the GITHUB_REPOSITORY_NAME output. + needs: [workflow-setup, lint-image] + runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}/indy-node-lint - needs: [workflow-setup, build-lint-image] - steps: + image: ghcr.io/${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}/node-lint + steps: - name: Check out code uses: actions/checkout@v2 - name: flake8 run: python3 -m flake8 + + build_release: + name: Build Release + needs: [workflow-setup, indy_node_tests, lint] + runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}/node-build:ubuntu-16-04 + steps: + - name: Check out code + uses: actions/checkout@v1 + + - name: Set Build Version + id: version + uses: ./.github/actions/set-version + with: + moduleName: indy_node + isDev: ${{ needs.workflow-setup.outputs.isDev }} + isRC: ${{ needs.workflow-setup.outputs.isRC }} + + - name: Build Deployment Package + run: | + mkdir -p /tmp/node-build + ./build-scripts/ubuntu-1604/build-indy-node.sh "/__w/indy-node/indy-node" "${{ steps.version.outputs.upstreamVer }}" "/tmp/node-build" "${{ steps.version.outputs.pkgVer }}" + + - uses: actions/upload-artifact@v2 + with: + name: node-deb + path: /tmp/node-build + retention-days: 5 + + build_3rd_party_dependencies: + name: Build 3rd Party Dependencies + needs: [workflow-setup, indy_node_tests, lint] + runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}/node-build:ubuntu-16-04 + steps: + - name: Check out code + uses: actions/checkout@v1 + + - name: Try load from cache. + id: third-party-dependencies + uses: actions/cache@v2 + with: + path: /tmp/third-party-dependencies + key: third-party-dependencies-${{ hashFiles('./build-scripts/ubuntu-1604/build-3rd-parties.sh') }} + + - name: Build 3rd party deployment packages + if: steps.third-party-dependencies.outputs.cache-hit != 'true' + run: | + mkdir -p ./build-scripts/ubuntu-1604/cache/3rd-party-dependencies/ + ./build-scripts/ubuntu-1604/build-3rd-parties.sh ./cache/3rd-party-dependencies + mv ./build-scripts/ubuntu-1604/cache/* /tmp/third-party-dependencies + + publish_artifacts: + name: Publish Artifacts + runs-on: ubuntu-20.04 + needs: [workflow-setup, build_release, build_3rd_party_dependencies] + if: needs.workflow-setup.outputs.publish == 'true' + env: + GITHUB_REF: ${{ needs.workflow-setup.outputs.GITHUB_REF }} + steps: + - name: Check out code + uses: actions/checkout@v1 + + - name: Setup JFrog CLI + uses: jfrog/setup-jfrog-cli@v1 + env: + JF_ARTIFACTORY_1: ${{ secrets.INDY_ARTIFACTORY_REPO_CONFIG }} + + - name: Ping Artifactory + run: | + # Test the connection to Ping the Hyperledger Artifactory server + # to ensure everything has been setup correctly. + jfrog rt ping + + - name: Download Node Artifacts from Pipeline Artifacts + uses: actions/download-artifact@v2 + with: + name: node-deb + path: to_publish + + - name: Publish Node Artifacts + uses: ./.github/actions/publish-deb + with: + sourceDirectory: /home/runner/work/indy-node/indy-node/to_publish + distribution: xenial + component: ${{ env.GITHUB_REF }} + + - name: Download 3rd Party Artifacts Dependencies from Cache + id: third-party-dependencies + uses: actions/cache@v2 + with: + path: /tmp/third-party-dependencies + key: third-party-dependencies-${{ hashFiles('./build-scripts/ubuntu-1604/build-3rd-parties.sh') }} + + - name: Publish 3rd Party Dependencies + uses: ./.github/actions/publish-deb + with: + sourceDirectory: /home/runner/tmp/third-party-dependencies + distribution: xenial + component: ${{ env.GITHUB_REF }} \ No newline at end of file diff --git a/.github/workflows/build/Dockerfile b/.github/workflows/build/Dockerfile index a53e03c44..37cbd0e82 100644 --- a/.github/workflows/build/Dockerfile +++ b/.github/workflows/build/Dockerfile @@ -1,10 +1,12 @@ FROM hyperledger/indy-core-baseci:0.0.3-master LABEL maintainer="Hyperledger " -RUN apt-get update -y && apt-get install -y \ +RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CE7709D068DB5E88 \ + && apt-get update -y \ + && apt-get install -y \ python3-nacl \ libindy-crypto=0.4.5 \ - libindy=1.15.0~1618-xenial \ + libindy=1.15.0~1625-xenial \ # rocksdb python wrapper libbz2-dev \ zlib1g-dev \ @@ -12,6 +14,16 @@ RUN apt-get update -y && apt-get install -y \ libsnappy-dev \ rocksdb=5.8.8 \ ursa=0.3.2-2 \ - jq +# Build dependencies + ruby \ + ruby-dev \ + rubygems \ + gcc \ + make \ +# zstd is needed for caching in github actions pipeline + zstd -RUN indy_image_clean +# install fpm +RUN gem install --no-ri --no-rdoc rake fpm + +RUN indy_image_clean \ No newline at end of file diff --git a/.github/workflows/lint/Dockerfile b/.github/workflows/lint/Dockerfile index d1870d403..4585a03eb 100644 --- a/.github/workflows/lint/Dockerfile +++ b/.github/workflows/lint/Dockerfile @@ -1,6 +1,6 @@ # Development FROM ubuntu:18.04 -LABEL maintainer="Kevin Griffin " +LABEL maintainer="Hyperledger " RUN apt-get update && apt-get dist-upgrade -y diff --git a/Jenkinsfile.ci b/Jenkinsfile.ci index 210d74386..1fc543ddd 100644 --- a/Jenkinsfile.ci +++ b/Jenkinsfile.ci @@ -98,7 +98,7 @@ def test(options=[:]) { try { if (options.useRunner) { - sh "PYTHONASYNCIODEBUG='0' $options.python runner.py --pytest \"$options.python -m pytest -l -vv\" --dir $options.testDir --output \"$options.resFile\" --test-only-slice \"$options.testOnlySlice\"" + sh "PYTHONASYNCIODEBUG='0' $options.python runner.py --pytest \"$options.python -m pytest -l -vv\" --dir $options.testDir --output \"$options.resFile\" --test-only-slice \"$options.testOnlySlice\" --nojunitxml" } else { sh "$options.python -m pytest -l -vv --junitxml=$options.resFile $options.testDir" } @@ -153,7 +153,6 @@ def tests = [ }, ].collect {k, v -> [k, v]} - def builds = [:] def _labels = labels.collect {k, v -> v} for (i = 0; i < _labels.size(); i++) { diff --git a/build-scripts/ubuntu-1604/build-3rd-parties.sh b/build-scripts/ubuntu-1604/build-3rd-parties.sh index 994945d0e..4b7311b73 100755 --- a/build-scripts/ubuntu-1604/build-3rd-parties.sh +++ b/build-scripts/ubuntu-1604/build-3rd-parties.sh @@ -41,5 +41,11 @@ function build_from_pypi { # build 3rd parties: # build_from_pypi # TODO duplicates list from Jenkinsfile.cd + +SCRIPT_PATH="${BASH_SOURCE[0]}" +pushd `dirname ${SCRIPT_PATH}` >/dev/null + build_from_pypi timeout-decorator 0.4.0 build_from_pypi distro 1.3.0 + +popd >/dev/null \ No newline at end of file diff --git a/ci/ubuntu.dockerfile b/ci/ubuntu.dockerfile index 8fa83c8b3..c4799e8d9 100644 --- a/ci/ubuntu.dockerfile +++ b/ci/ubuntu.dockerfile @@ -5,6 +5,9 @@ ARG uid=1000 ARG user=indy ARG venv=venv +# Update Sovrin signing key +RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CE7709D068DB5E88 + RUN apt-get update -y && apt-get install -y \ python3-nacl \ ursa=0.3.2-2 \ diff --git a/indy_node/__metadata__.py b/indy_node/__metadata__.py index f8fa05d90..1603cb026 100644 --- a/indy_node/__metadata__.py +++ b/indy_node/__metadata__.py @@ -47,7 +47,7 @@ def load_manifest(manifest_file: str = MANIFEST_FILE) -> Any: def set_manifest(manifest: Any, manifest_file: str = MANIFEST_FILE): with open(manifest_file, 'w') as _f: - json.dump(manifest, _f) + json.dump(manifest, _f, sort_keys=True) _f.write('\n') diff --git a/runner.py b/runner.py index c69dfce9d..d5ba4d4ac 100644 --- a/runner.py +++ b/runner.py @@ -14,7 +14,7 @@ def get_first_level_path(path): return "/".join(parts[:3]) # /test/ -def run(pytest, output_file, repeatUntilFailure, testDir, test_slice): +def run(pytest, output_file, repeatUntilFailure, testDir, test_slice, test_results): if repeatUntilFailure: log("'repeatUntilFailure' is set") log("Is going to repeat the test suite until failure") @@ -45,7 +45,8 @@ def run(pytest, output_file, repeatUntilFailure, testDir, test_slice): first_level_tests.add(p) first_level_tests = sorted(list(first_level_tests)) test_list_sliced = first_level_tests[test_slice[0] - 1::test_slice[1]] - log("Found {} tests in {} modules, sliced {} test modules".format(len(testList), len(first_level_tests), len(test_list_sliced))) + log("Found {} tests in {} modules, sliced {} test modules". + format(len(testList), len(first_level_tests), len(test_list_sliced))) if not testList: m = re.search("errors during collection", collectedData) if m: @@ -69,9 +70,12 @@ def run(pytest, output_file, repeatUntilFailure, testDir, test_slice): errorTestPat = re.compile('____ (ERROR.+) ____') while True: for i, tests in enumerate(test_list_sliced): - # testRep = '{}.rep'.format(test.split("/")[-1]) + if test_results: + testResults = '--junitxml={}-test-results.xml'.format(tests.split("/")[-1]) + pytest_cmd = '{} {} {} > {}'.format(pytest, testResults, tests, testRep) + else: + pytest_cmd = '{} {} > {}'.format(pytest, tests, testRep) testStartTime = time.time() - pytest_cmd = '{} {} > {}'.format(pytest, tests, testRep) log(pytest_cmd) r = os.system(pytest_cmd) testExecutionTime = time.time() - testStartTime @@ -207,12 +211,16 @@ def parse_test_slice(x): default=(1, 1), help='example: `1/3` -- run only first third part of the tests, \ it is needed only for parallel testing') + parser.add_argument('--nojunitxml', + help='do not record junitxml test results', default=False, action="store_true") + args = parser.parse_args() r = run( pytest=args.pytest, output_file=args.output if not args.nooutput else None, repeatUntilFailure=args.repeatUntilFailure, testDir=args.dir, - test_slice=args.test_slice + test_slice=args.test_slice, + test_results=not args.nojunitxml ) sys.exit(0 if r == 0 else 1)