container: Revamp container image installation #4495
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Tests | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - main | |
| - "test/**" | |
| schedule: | |
| - cron: "2 0 * * *" # Run every day at 02:00 UTC. | |
| workflow_dispatch: | |
| permissions: | |
| packages: write | |
| env: | |
| REGISTRY_USER: ${{ github.actor }} | |
| REGISTRY_PASSWORD: ${{ github.token }} | |
| IMAGE_REGISTRY: ghcr.io/${{ github.repository_owner }} | |
| QT_SELECT: "qt6" | |
| # Disable multiple concurrent runs on the same branch | |
| # When a new CI build is triggered, it will cancel the | |
| # other in-progress ones (for the same branch) | |
| concurrency: | |
| group: ${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| run-lint: | |
| runs-on: ubuntu-latest | |
| container: | |
| image: debian:bookworm | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install dev. dependencies | |
| run: |- | |
| apt-get update | |
| apt-get install -y git make python3 python3-poetry --no-install-recommends | |
| poetry install --only lint,test | |
| - name: Run linters to enforce code style | |
| run: poetry run make lint | |
| - name: Check that the QA script is up to date with the docs | |
| run: "./dev_scripts/qa.py --check-refs" | |
| # This is already built daily by the "build.yml" file | |
| # But we also want to include this in the checks that run on each push. | |
| build-container-image: | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Get current date | |
| id: date | |
| run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT | |
| - name: Cache container image | |
| id: cache-container-image | |
| uses: actions/cache@v4 | |
| with: | |
| key: v2-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/common.py', 'dangerzone/conversion/doc_to_pixels.py', 'dangerzone/conversion/pixels_to_pdf.py', 'poetry.lock', 'gvisor_wrapper/entrypoint.py') }} | |
| path: |- | |
| share/container.tar.gz | |
| share/image-id.txt | |
| - name: Build Dangerzone container image | |
| if: ${{ steps.cache-container-image.outputs.cache-hit != 'true' }} | |
| run: | | |
| sudo apt-get install -y python3-poetry | |
| python3 ./install/common/build-image.py | |
| - name: Upload container image | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: container.tar.gz | |
| path: share/container.tar.gz | |
| download-tessdata: | |
| name: Download and cache Tesseract data | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Cache Tessdata | |
| id: cache-tessdata | |
| uses: actions/cache@v4 | |
| with: | |
| path: share/tessdata/ | |
| key: v1-tessdata-${{ hashFiles('./install/common/download-tessdata.py') }} | |
| enableCrossOsArchive: true | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Download Tessdata | |
| run: |- | |
| if [ -f "share/tessdata" ]; then | |
| echo "Already cached, skipping" | |
| else | |
| python3 ./install/common/download-tessdata.py | |
| fi | |
| windows: | |
| runs-on: windows-latest | |
| needs: | |
| - download-tessdata | |
| env: | |
| DUMMY_CONVERSION: 1 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - run: pip install poetry | |
| - run: poetry install | |
| - name: Restore cached tessdata | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: share/tessdata/ | |
| enableCrossOsArchive: true | |
| fail-on-cache-miss: true | |
| key: v1-tessdata-${{ hashFiles('./install/common/download-tessdata.py') }} | |
| - name: Run CLI tests | |
| run: poetry run make test | |
| # Taken from: https://github.com/orgs/community/discussions/27149#discussioncomment-3254829 | |
| - name: Set path for candle and light | |
| run: echo "C:\Program Files (x86)\WiX Toolset v3.14\bin" >> $GITHUB_PATH | |
| shell: bash | |
| - name: Build the MSI installer | |
| # NOTE: This also builds the .exe internally. | |
| run: poetry run .\install\windows\build-app.bat | |
| - name: Upload MSI installer | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: Dangerzone.msi | |
| path: "dist/Dangerzone.msi" | |
| if-no-files-found: error | |
| compression-level: 0 | |
| macOS: | |
| name: "macOS (${{ matrix.arch }})" | |
| runs-on: ${{ matrix.runner }} | |
| needs: | |
| - download-tessdata | |
| strategy: | |
| matrix: | |
| include: | |
| - runner: macos-latest # CPU type: Apple Silicon (M1) | |
| arch: arch64 | |
| - runner: macos-13 # CPU type: Intel x86_64 | |
| arch: x86_64 | |
| env: | |
| DUMMY_CONVERSION: 1 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Restore cached tessdata | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: share/tessdata/ | |
| enableCrossOsArchive: true | |
| fail-on-cache-miss: true | |
| key: v1-tessdata-${{ hashFiles('./install/common/download-tessdata.py') }} | |
| - run: pip install poetry | |
| - run: poetry install | |
| - name: Run CLI tests | |
| run: poetry run make test | |
| - name: Build macOS app | |
| run: poetry run python ./install/macos/build-app.py | |
| - name: Upload macOS app | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: Dangerzone-${{ matrix.arch }}.app | |
| path: "dist/Dangerzone.app" | |
| if-no-files-found: error | |
| compression-level: 0 | |
| build-deb: | |
| needs: | |
| - build-container-image | |
| name: "build-deb (${{ matrix.distro }} ${{ matrix.version }})" | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| include: | |
| - distro: ubuntu | |
| version: "20.04" | |
| - distro: ubuntu | |
| version: "22.04" | |
| - distro: ubuntu | |
| version: "24.04" | |
| - distro: ubuntu | |
| version: "24.10" | |
| - distro: debian | |
| version: bullseye | |
| - distro: debian | |
| version: bookworm | |
| - distro: debian | |
| version: trixie | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Login to GHCR | |
| run: | | |
| echo ${{ github.token }} | podman login ghcr.io -u USERNAME --password-stdin | |
| - name: Get the dev environment | |
| run: | | |
| ./dev_scripts/env.py \ | |
| --distro ${{ matrix.distro }} \ | |
| --version ${{ matrix.version }} \ | |
| build-dev --sync | |
| - name: Get current date | |
| id: date | |
| run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT | |
| - name: Restore container cache | |
| uses: actions/cache/restore@v4 | |
| with: | |
| key: v2-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/common.py', 'dangerzone/conversion/doc_to_pixels.py', 'dangerzone/conversion/pixels_to_pdf.py', 'poetry.lock', 'gvisor_wrapper/entrypoint.py') }} | |
| path: |- | |
| share/container.tar.gz | |
| share/image-id.txt | |
| fail-on-cache-miss: true | |
| - name: Build Dangerzone .deb | |
| run: | | |
| ./dev_scripts/env.py --distro ${{ matrix.distro }} \ | |
| --version ${{ matrix.version }} \ | |
| run --dev --no-gui ./dangerzone/install/linux/build-deb.py | |
| - name: Upload Dangerzone .deb | |
| if: matrix.distro == 'debian' && matrix.version == 'bookworm' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: dangerzone.deb | |
| path: "deb_dist/dangerzone_*_*.deb" | |
| if-no-files-found: error | |
| compression-level: 0 | |
| install-deb: | |
| name: "install-deb (${{ matrix.distro }} ${{ matrix.version }})" | |
| runs-on: ubuntu-latest | |
| needs: | |
| - build-deb | |
| strategy: | |
| matrix: | |
| include: | |
| - distro: ubuntu | |
| version: "20.04" | |
| - distro: ubuntu | |
| version: "22.04" | |
| - distro: ubuntu | |
| version: "24.04" | |
| - distro: ubuntu | |
| version: "24.10" | |
| - distro: debian | |
| version: bullseye | |
| - distro: debian | |
| version: bookworm | |
| - distro: debian | |
| version: trixie | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Download Dangerzone .deb | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: dangerzone.deb | |
| path: "deb_dist/" | |
| - name: Build end-user environment | |
| run: | | |
| ./dev_scripts/env.py --distro ${{ matrix.distro }} \ | |
| --version ${{ matrix.version }} \ | |
| build | |
| - name: Run a test command | |
| run: | | |
| ./dev_scripts/env.py --distro ${{ matrix.distro }} \ | |
| --version ${{ matrix.version }} \ | |
| run dangerzone-cli dangerzone/tests/test_docs/sample-pdf.pdf --ocr-lang eng | |
| - name: Check that the Dangerzone GUI imports work | |
| run: | | |
| ./dev_scripts/env.py --distro ${{ matrix.distro }} \ | |
| --version ${{ matrix.version }} \ | |
| run dangerzone --help | |
| build-install-rpm: | |
| name: "build-install-rpm (${{ matrix.distro }} ${{matrix.version}})" | |
| runs-on: ubuntu-latest | |
| needs: | |
| - build-container-image | |
| strategy: | |
| matrix: | |
| distro: ["fedora"] | |
| version: ["40", "41"] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Login to GHCR | |
| run: | | |
| echo ${{ github.token }} | podman login ghcr.io -u USERNAME --password-stdin | |
| - name: Get the dev environment | |
| run: | | |
| ./dev_scripts/env.py \ | |
| --distro ${{ matrix.distro }} \ | |
| --version ${{ matrix.version }} \ | |
| build-dev --sync | |
| - name: Get current date | |
| id: date | |
| run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT | |
| - name: Restore container image | |
| uses: actions/cache/restore@v4 | |
| with: | |
| key: v2-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/common.py', 'dangerzone/conversion/doc_to_pixels.py', 'dangerzone/conversion/pixels_to_pdf.py', 'poetry.lock', 'gvisor_wrapper/entrypoint.py') }} | |
| path: |- | |
| share/container.tar.gz | |
| share/image-id.txt | |
| fail-on-cache-miss: true | |
| - name: Build Dangerzone .rpm | |
| run: | | |
| ./dev_scripts/env.py --distro ${{ matrix.distro }} --version ${{ matrix.version }} \ | |
| run --dev --no-gui ./dangerzone/install/linux/build-rpm.py | |
| - name: Upload Dangerzone .rpm | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: dangerzone-${{ matrix.distro }}-${{ matrix.version }}.rpm | |
| path: "dist/dangerzone-*.x86_64.rpm" | |
| if-no-files-found: error | |
| compression-level: 0 | |
| # Reclaim some space in this step, now that the dev environment is no | |
| # longer necessary. Previously, we encountered out-of-space issues while | |
| # running this CI job. | |
| - name: Reclaim some storage space | |
| run: podman system reset -f | |
| - name: Build end-user environment | |
| run: | | |
| ./dev_scripts/env.py --distro ${{ matrix.distro }} \ | |
| --version ${{ matrix.version }} \ | |
| build | |
| - name: Run a test command | |
| run: | | |
| ./dev_scripts/env.py --distro ${{ matrix.distro }} --version ${{ matrix.version }} \ | |
| run dangerzone-cli dangerzone/tests/test_docs/sample-pdf.pdf --ocr-lang eng | |
| - name: Check that the Dangerzone GUI imports work | |
| run: | | |
| ./dev_scripts/env.py --distro ${{ matrix.distro }} --version ${{ matrix.version }} \ | |
| run dangerzone --help | |
| run-tests: | |
| name: "run tests (${{ matrix.distro }} ${{ matrix.version }})" | |
| runs-on: ubuntu-latest | |
| needs: | |
| - build-container-image | |
| - download-tessdata | |
| strategy: | |
| matrix: | |
| include: | |
| - distro: ubuntu | |
| version: "20.04" | |
| - distro: ubuntu | |
| version: "22.04" | |
| - distro: ubuntu | |
| version: "24.04" | |
| - distro: ubuntu | |
| version: "24.10" | |
| - distro: debian | |
| version: bullseye | |
| - distro: debian | |
| version: bookworm | |
| - distro: debian | |
| version: trixie | |
| - distro: fedora | |
| version: "40" | |
| - distro: fedora | |
| version: "41" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Login to GHCR | |
| run: | | |
| echo ${{ github.token }} | podman login ghcr.io -u USERNAME --password-stdin | |
| - name: Get current date | |
| id: date | |
| run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT | |
| - name: Get the dev environment | |
| run: | | |
| ./dev_scripts/env.py \ | |
| --distro ${{ matrix.distro }} \ | |
| --version ${{ matrix.version }} \ | |
| build-dev --sync | |
| - name: Restore container image | |
| uses: actions/cache/restore@v4 | |
| with: | |
| key: v2-${{ steps.date.outputs.date }}-${{ hashFiles('Dockerfile', 'dangerzone/conversion/common.py', 'dangerzone/conversion/doc_to_pixels.py', 'dangerzone/conversion/pixels_to_pdf.py', 'poetry.lock', 'gvisor_wrapper/entrypoint.py') }} | |
| path: |- | |
| share/container.tar.gz | |
| share/image-id.txt | |
| fail-on-cache-miss: true | |
| - name: Restore cached tessdata | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: share/tessdata/ | |
| enableCrossOsArchive: true | |
| fail-on-cache-miss: true | |
| key: v1-tessdata-${{ hashFiles('./install/common/download-tessdata.py') }} | |
| - name: Setup xvfb (Linux) | |
| run: | | |
| # Stuff copied wildly from several stackoverflow posts | |
| sudo apt-get install -y xvfb libxkbcommon-x11-0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-xinerama0 libxcb-xinput0 libxcb-xfixes0 libxcb-shape0 libglib2.0-0 libgl1-mesa-dev '^libxcb.*-dev' libx11-xcb-dev libglu1-mesa-dev libxrender-dev libxi-dev libxkbcommon-dev libxkbcommon-x11-dev | |
| # start xvfb in the background | |
| sudo /usr/bin/Xvfb $DISPLAY -screen 0 1280x1024x24 & | |
| - name: Run CI tests | |
| run: |- | |
| # Pass the -ac Xserver flag, to disable host-based access controls. | |
| # This should be used ONLY for testing [1]. If we don't pass this | |
| # flag, the Podman container is not authorized [2] to access the Xvfb | |
| # server. | |
| # | |
| # [1] From https://www.x.org/releases/X11R6.7.0/doc/Xserver.1.html#sect4: | |
| # | |
| # disables host-based access control mechanisms. Enables access by | |
| # any host, and permits any host to modify the access control | |
| # list. Use with extreme caution. This option exists primarily for | |
| # running test suites remotely. | |
| # | |
| # [2] Fails with "Authorization required, but no authorization | |
| # protocol specified". However, we have verified with strace(1) | |
| # that the command in the Podman container can read the Xauthority | |
| # file successfully. | |
| xvfb-run -s '-ac' ./dev_scripts/env.py --distro ${{ matrix.distro }} --version ${{ matrix.version }} run --dev \ | |
| bash -c 'cd dangerzone; poetry run make test' |