From 35a776a349bbbe7a5701a4fc3ecb7098f6865669 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 14 Jan 2025 09:51:09 +0000 Subject: [PATCH] Migrate to charmcraft 3 poetry plugin (#28) data-platform-workflows and charmcraftcache are migrating from charmcraft 2 to 3 to enable 24.04 based charms & to enable use of the poetry plugin, which fixes several longstanding issues with charmcraft (e.g. https://github.com/canonical/charmcraft/issues/1077) --- .github/workflows/ci.yaml | 3 +- .github/workflows/release.yaml | 7 +- charmcraft.yaml | 99 ++++++++++++++++----- poetry.lock | 8 +- pyproject.toml | 7 +- tests/integration/app-charm/charmcraft.yaml | 14 ++- tox.ini | 4 - 7 files changed, 96 insertions(+), 46 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0e2484d..bac3f4d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,9 +42,10 @@ jobs: build: name: Build charms - uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v24.0.6 + uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v26.0.0 with: cache: true + charmcraft-snap-channel: latest/beta/data-platform # TODO: remove after charmcraft 3.3 stable release path-to-charm-directory: ${{ matrix.path }} strategy: matrix: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 0a44fcd..25d0760 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -35,9 +35,9 @@ jobs: build: name: Build charm - uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v24.0.6 + uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v26.0.0 with: - cache: false + charmcraft-snap-channel: latest/beta/data-platform # TODO: remove after charmcraft 3.3 stable release release: name: Release to Charmhub @@ -45,8 +45,9 @@ jobs: - lib-check - ci-tests - build - uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v24.0.6 + uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v26.0.0 with: + charmcraft-snap-channel: latest/beta/data-platform # TODO: remove after charmcraft 3.3 stable release channel: latest/edge artifact-prefix: ${{ needs.build.outputs.artifact-prefix }} secrets: diff --git a/charmcraft.yaml b/charmcraft.yaml index 39c787f..6a76a21 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -1,31 +1,86 @@ -# Copyright 2024 Canonical Ltd. +# Copyright 2022 Canonical Ltd. # See LICENSE file for licensing details. type: charm +platforms: + ubuntu@22.04:amd64: +# Files implicitly created by charmcraft without a part: +# - dispatch (https://github.com/canonical/charmcraft/pull/1898) +# - manifest.yaml +# (https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/services/package.py#L259) +# Files implicitly copied/"primed" by charmcraft without a part: +# - actions.yaml, config.yaml, metadata.yaml +# (https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/services/package.py#L290-L293 +# https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/services/package.py#L156-L157) parts: - charm: - build-snaps: - - rustup + # "poetry-deps" part name is a magic constant + # https://github.com/canonical/craft-parts/pull/901 + poetry-deps: + plugin: nil build-packages: - - pkg-config - - libffi-dev - - libssl-dev + - curl override-build: | - # get latest rust stable toolchain - rustup default stable + # Use environment variable instead of `--break-system-packages` to avoid failing on older + # versions of pip that do not recognize `--break-system-packages` + # `--user` needed (in addition to `--break-system-packages`) for Ubuntu >=24.04 + PIP_BREAK_SYSTEM_PACKAGES=true python3 -m pip install --user --upgrade pip==24.3.1 # renovate: charmcraft-pip-latest - # Convert subset of poetry.lock to requirements.txt - curl -sSL https://install.python-poetry.org | python3 - - /root/.local/bin/poetry self add poetry-plugin-export - /root/.local/bin/poetry export --only main,charm-libs --output requirements.txt + # Use uv to install poetry so that a newer version of Python can be installed if needed by poetry + curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.5.15/uv-installer.sh | sh # renovate: charmcraft-uv-latest + # poetry 2.0.0 requires Python >=3.9 + if ! "$HOME/.local/bin/uv" python find '>=3.9' + then + # Use first Python version that is >=3.9 and available in an Ubuntu LTS + # (to reduce the number of Python versions we use) + "$HOME/.local/bin/uv" python install 3.10.12 # renovate: charmcraft-python-ubuntu-22.04 + fi + "$HOME/.local/bin/uv" tool install --no-python-downloads --python '>=3.9' poetry==2.0.0 --with poetry-plugin-export==1.8.0 # renovate: charmcraft-poetry-latest + + ln -sf "$HOME/.local/bin/poetry" /usr/local/bin/poetry + # "charm-poetry" part name is arbitrary; use for consistency + # Avoid using "charm" part name since that has special meaning to charmcraft + charm-poetry: + # By default, the `poetry` plugin creates/primes these directories: + # - lib, src + # (https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/parts/plugins/_poetry.py#L76-L78) + # - venv + # (https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/parts/plugins/_poetry.py#L95 + # https://github.com/canonical/craft-parts/blob/afb0d652eb330b6aaad4f40fbd6e5357d358de47/craft_parts/plugins/base.py#L270) + plugin: poetry + source: . + after: + - poetry-deps + poetry-export-extra-args: ['--only', 'main,charm-libs'] + build-packages: + - libffi-dev # Needed to build Python dependencies with Rust from source + - libssl-dev # Needed to build Python dependencies with Rust from source + - pkg-config # Needed to build Python dependencies with Rust from source + override-build: | + # Workaround for https://github.com/canonical/charmcraft/issues/2068 + # rustup used to install rustc and cargo, which are needed to build Python dependencies with Rust from source + if [[ "$CRAFT_PLATFORM" == ubuntu@20.04:* || "$CRAFT_PLATFORM" == ubuntu@22.04:* ]] + then + snap install rustup --classic + else + apt-get install rustup -y + fi + + # If Ubuntu version < 24.04, rustup was installed from snap instead of from the Ubuntu + # archive—which means the rustup version could be updated at any time. Print rustup version + # to build log to make changes to the snap's rustup version easier to track + rustup --version + + # rpds-py (Python package) >=0.19.0 requires rustc >=1.76, which is not available in the + # Ubuntu 22.04 archive. Install rustc and cargo using rustup instead of the Ubuntu archive + rustup set profile minimal + rustup default 1.83.0 # renovate: charmcraft-rust-latest craftctl default - charm-strict-dependencies: true - charm-requirements: [requirements.txt] -bases: - - build-on: - - name: "ubuntu" - channel: "22.04" - run-on: - - name: "ubuntu" - channel: "22.04" + # Include requirements.txt in *.charm artifact for easier debugging + cp requirements.txt "$CRAFT_PART_INSTALL/requirements.txt" + # "files" part name is arbitrary; use for consistency + files: + plugin: dump + source: . + prime: + - LICENSE diff --git a/poetry.lock b/poetry.lock index 49cd129..e92af47 100644 --- a/poetry.lock +++ b/poetry.lock @@ -511,6 +511,7 @@ files = [ {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:761817a3377ef15ac23cd7834715081791d4ec77f9297ee694ca1ee9c2c7e5eb"}, {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3c672a53c0fb4725a29c303be906d3c1fa99c32f58abe008a82705f9ee96f40b"}, {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:4ac4c9f37eba52cb6fbeaf5b59c152ea976726b865bd4cf87883a7e7006cc543"}, + {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:60eb32934076fa07e4316b7b2742fa52cbb190b42c2df2863dbc4230a0a9b385"}, {file = "cryptography-44.0.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ed3534eb1090483c96178fcb0f8893719d96d5274dfde98aa6add34614e97c8e"}, {file = "cryptography-44.0.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:f3f6fdfa89ee2d9d496e2c087cebef9d4fcbb0ad63c40e821b39f74bf48d9c5e"}, {file = "cryptography-44.0.0-cp37-abi3-win32.whl", hash = "sha256:eb33480f1bad5b78233b0ad3e1b0be21e8ef1da745d8d2aecbb20671658b9053"}, @@ -521,6 +522,7 @@ files = [ {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:c5eb858beed7835e5ad1faba59e865109f3e52b3783b9ac21e7e47dc5554e289"}, {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f53c2c87e0fb4b0c00fa9571082a057e37690a8f12233306161c8f4b819960b7"}, {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:9e6fc8a08e116fb7c7dd1f040074c9d7b51d74a8ea40d4df2fc7aa08b76b9e6c"}, + {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9abcc2e083cbe8dde89124a47e5e53ec38751f0d7dfd36801008f316a127d7ba"}, {file = "cryptography-44.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:d2436114e46b36d00f8b72ff57e598978b37399d2786fd39793c36c6d5cb1c64"}, {file = "cryptography-44.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a01956ddfa0a6790d594f5b34fc1bfa6098aca434696a03cfdbe469b8ed79285"}, {file = "cryptography-44.0.0-cp39-abi3-win32.whl", hash = "sha256:eca27345e1214d1b9f9490d200f9db5a874479be914199194e746c893788d417"}, @@ -1638,8 +1640,8 @@ pyyaml = "*" [package.source] type = "git" url = "https://github.com/canonical/data-platform-workflows" -reference = "v24.0.6" -resolved_reference = "11c673f692893a15d15ee63469420e91f91f8a95" +reference = "v26.0.0" +resolved_reference = "92d0a9f28a22c57b5965866c22f65b5021d3b115" subdirectory = "python/pytest_plugins/pytest_operator_cache" [[package]] @@ -2233,4 +2235,4 @@ files = [ [metadata] lock-version = "2.1" python-versions = ">=3.10,<4.0" -content-hash = "eb187ed095687a7c9ef38f06c9722fe9948d0a5f1b4a750887cbeb08a156fe3b" +content-hash = "d213083d8a1d2fd8cbb943d2472ab68d4dc88babafd07b41f64e2bcbc014cf89" diff --git a/pyproject.toml b/pyproject.toml index 9608b43..73a1057 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,11 +20,8 @@ target-version = ["py310"] profile = "black" [tool.poetry] -name = "karapace-operator" -version = "1.0" -description = "karapace-operator" -authors = [] package-mode = false +requires-poetry = ">=2.0.0" [tool.poetry.dependencies] python = ">=3.10,<4.0" @@ -87,7 +84,7 @@ pure-sasl = ">=0.5" requests = ">2.25" pyyaml = "^6.0.1" -pytest-operator-cache = { git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.6", subdirectory = "python/pytest_plugins/pytest_operator_cache" } +pytest-operator-cache = { git = "https://github.com/canonical/data-platform-workflows", tag = "v26.0.0", subdirectory = "python/pytest_plugins/pytest_operator_cache" } # To be enabled if we are using groups on integration tests # pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v7", subdirectory = "python/pytest_plugins/pytest_operator_groups"} diff --git a/tests/integration/app-charm/charmcraft.yaml b/tests/integration/app-charm/charmcraft.yaml index 1235c6e..e7ff5f1 100644 --- a/tests/integration/app-charm/charmcraft.yaml +++ b/tests/integration/app-charm/charmcraft.yaml @@ -1,11 +1,9 @@ -# Copyright 2024 Canonical Ltd. +# Copyright 2022 Canonical Ltd. # See LICENSE file for licensing details. type: charm -bases: - - build-on: - - name: "ubuntu" - channel: "22.04" - run-on: - - name: "ubuntu" - channel: "22.04" +platforms: + ubuntu@22.04:amd64: +parts: + charm: + plugin: charm diff --git a/tox.ini b/tox.ini index c9a6306..d2a6b4c 100644 --- a/tox.ini +++ b/tox.ini @@ -91,10 +91,6 @@ commands = [testenv:integration-{charm,password-rotation,tls,provider}] description = Run integration tests -set_env = - {[testenv]set_env} - # Workaround for https://github.com/python-poetry/poetry/issues/6958 - POETRY_INSTALLER_PARALLEL = false pass_env = {[testenv]pass_env} CI