diff --git a/.fmf/version b/.fmf/version new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/.fmf/version @@ -0,0 +1 @@ +1 diff --git a/.github/workflows/greenboot-ci.yaml b/.github/workflows/greenboot-ci.yaml new file mode 100644 index 0000000..920e2cf --- /dev/null +++ b/.github/workflows/greenboot-ci.yaml @@ -0,0 +1,37 @@ +--- +name: Greenboot Test + +on: + pull_request: + types: [opened, synchronize, reopened] + +jobs: + fedora-41-bootc-qcow2: + continue-on-error: true + runs-on: ubuntu-latest + + steps: + - name: Get information for pull request + uses: octokit/request-action@v2.x + id: pr-api + with: + route: GET /repos/${{ github.repository }}/pulls/${{ github.event.number }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Run the tests + uses: sclorg/testing-farm-as-github-action@v3.1.2 + with: + compose: Fedora-41 + api_key: ${{ secrets.TF_API_KEY }} + git_url: ${{ fromJson(steps.pr-api.outputs.data).head.repo.html_url }} + git_ref: ${{ fromJson(steps.pr-api.outputs.data).head.ref }} + github_token: ${{ secrets.PAT }} + update_pull_request_status: true + pull_request_status_name: "f41-bootc-qcow2" + tmt_context: "arch=x86_64;distro=fedora-41" + tmt_plan_regex: greenboot-bootc-qcow2 + tf_scope: private + secrets: "QUAY_USERNAME=${{ secrets.QUAY_USERNAME }};QUAY_PASSWORD=${{ secrets.QUAY_PASSWORD }};STAGE_REDHAT_IO_USERNAME=${{ secrets.STAGE_REDHAT_IO_USERNAME }};STAGE_REDHAT_IO_TOKEN=${{ secrets.STAGE_REDHAT_IO_TOKEN }}" + variables: "ARCH=x86_64" + timeout: 90 diff --git a/tests/greenboot-bootc-qcow2.sh b/tests/greenboot-bootc-qcow2.sh new file mode 100755 index 0000000..60349b5 --- /dev/null +++ b/tests/greenboot-bootc-qcow2.sh @@ -0,0 +1,326 @@ +#!/bin/bash +set -euox pipefail + +# Dumps details about the instance running the CI job. +echo -e "\033[0;36m" +cat << EOF +------------------------------------------------------------------------------ +CI MACHINE SPECS +------------------------------------------------------------------------------ + Hostname: $(uname -n) + User: $(whoami) + CPUs: $(nproc) + RAM: $(free -m | grep -oP '\d+' | head -n 1) MB + DISK: $(df --output=size -h / | sed '1d;s/[^0-9]//g') GB + ARCH: $(uname -m) + KERNEL: $(uname -r) +------------------------------------------------------------------------------ +EOF +echo -e "\033[0m" + +# Get OS info +source /etc/os-release + +# Setup variables +TEST_UUID=qcow2-$((1 + RANDOM % 1000000)) +TEMPDIR=$(mktemp -d) +GUEST_ADDRESS=192.168.100.50 +SSH_OPTIONS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5) +SSH_KEY=key/ostree_key +SSH_KEY_PUB=$(cat "${SSH_KEY}".pub) +EDGE_USER=core +EDGE_USER_PASSWORD=foobar + +case "${ID}-${VERSION_ID}" in + "fedora-41") + OS_VARIANT="fedora-unknown" + BASE_IMAGE_URL="quay.io/fedora/fedora-bootc:41" + BIB_URL="quay.io/centos-bootc/bootc-image-builder:latest" + ;; + "centos-9") + OS_VARIANT="centos-stream9" + BASE_IMAGE_URL="quay.io/centos-bootc/centos-bootc:stream9" + BIB_URL="quay.io/centos-bootc/bootc-image-builder:latest" + ;; + "rhel-9.6") + OS_VARIANT="rhel9-unknown" + BASE_IMAGE_URL="registry.stage.redhat.io/rhel9/rhel-bootc:9.6" + BIB_URL="registry.stage.redhat.io/rhel9/bootc-image-builder:9.6" + ;; + *) + echo "unsupported distro: ${ID}-${VERSION_ID}" + exit 1;; +esac + +# Colorful output. +function greenprint { + echo -e "\033[1;32m${1}\033[0m" +} + +check_result () { + greenprint "🎏 Checking for test result" + if [[ $RESULTS == 1 ]]; then + greenprint "💚 Success" + else + greenprint "❌ Failed" + clean_up + exit 1 + fi +} + +# Wait for the ssh server up to be. +wait_for_ssh_up () { + SSH_STATUS=$(sudo ssh "${SSH_OPTIONS[@]}" -i "${SSH_KEY}" ${EDGE_USER}@"${1}" '/bin/bash -c "echo -n READY"') + if [[ $SSH_STATUS == READY ]]; then + echo 1 + else + echo 0 + fi +} + +########################################################### +## +## Prepare before run test +## +########################################################### +greenprint "Installing required packages" +sudo dnf install -y podman qemu-img firewalld qemu-kvm libvirt-client libvirt-daemon-kvm libvirt-daemon virt-install rpmdevtools + +# Start firewalld +greenprint "Start firewalld" +sudo systemctl enable --now firewalld + +# Check ostree_key permissions +KEY_PERMISSION_PRE=$(stat -L -c "%a %G %U" key/ostree_key | grep -oP '\d+' | head -n 1) +echo -e "${KEY_PERMISSION_PRE}" +if [[ "${KEY_PERMISSION_PRE}" != "600" ]]; then + greenprint "💡 File permissions too open...Changing to 600" + chmod 600 ./key/ostree_key +fi + +# Setup libvirt +greenprint "Starting libvirt service and configure libvirt network" +sudo tee /etc/polkit-1/rules.d/50-libvirt.rules > /dev/null << EOF +polkit.addRule(function(action, subject) { + if (action.id == "org.libvirt.unix.manage" && + subject.isInGroup("adm")) { + return polkit.Result.YES; + } +}); +EOF +sudo systemctl start libvirtd +sudo virsh list --all > /dev/null +sudo tee /tmp/integration.xml > /dev/null << EOF + + integration + 1c8fe98c-b53a-4ca4-bbdb-deb0f26b3579 + + + + + + + + + + + + + + + + + + + + + +EOF +if ! sudo virsh net-info integration > /dev/null 2>&1; then + sudo virsh net-define /tmp/integration.xml +fi +if [[ $(sudo virsh net-info integration | grep 'Active' | awk '{print $2}') == 'no' ]]; then + sudo virsh net-start integration +fi + +########################################################### +## +## Build greenboot rpm packages +## +########################################################### +greenprint "Building greenboot packages" +pushd .. && \ +shopt -s extglob +version=$(cat greenboot.spec |grep Version|awk '{print $2}') +rm -rf greenboot-${version}/ rpmbuild/ +mkdir -p rpmbuild/BUILD rpmbuild/RPMS rpmbuild/SOURCES rpmbuild/SPECS rpmbuild/SRPMS +mkdir greenboot-${version} +cp -r !(rpmbuild|greenboot-${version}|build.sh) greenboot-${version}/ +tar -cvf v${version}.tar.gz greenboot-${version}/ +mv v${version}.tar.gz rpmbuild/SOURCES/ +rpmbuild -bb --define="_topdir ${PWD}/rpmbuild" greenboot.spec +chmod +x rpmbuild/RPMS/x86_64/*.rpm && \ +cp rpmbuild/RPMS/x86_64/*.rpm tests/ && popd + +########################################################### +## +## Build bootc container with greenboot installed +## +########################################################### +greenprint "Building rhel-edge-bootc container" +podman login quay.io -u ${QUAY_USERNAME} -p ${QUAY_PASSWORD} +podman login registry.stage.redhat.io -u ${STAGE_REDHAT_IO_USERNAME} -p ${STAGE_REDHAT_IO_TOKEN} +tee Containerfile > /dev/null << EOF +FROM ${BASE_IMAGE_URL} +# Copy the local RPM files into the container +COPY greenboot-*.rpm /tmp/ +RUN dnf install -y \ + /tmp/greenboot-*.rpm && \ + systemctl enable greenboot-grub2-set-counter \ + greenboot-grub2-set-success.service greenboot-healthcheck.service \ + greenboot-loading-message.service greenboot-rpm-ostree-grub2-check-fallback.service \ + redboot-auto-reboot.service redboot-task-runner.service redboot.target +# Clean up by removing the local RPMs if desired +RUN rm -f /tmp/greenboot-*.rpm +EOF +podman build --retry=5 --retry-delay=10 -t quay.io/${QUAY_USERNAME}/greenboot-bootc:${TEST_UUID} -f Containerfile . +greenprint "Pushing greenboot-bootc container to quay.io" +podman push quay.io/${QUAY_USERNAME}/greenboot-bootc:${TEST_UUID} + +########################################################### +## +## BIB to convert bootc container to qcow2/iso images +## +########################################################### +greenprint "Using BIB to convert container to qcow2" +tee config.json > /dev/null << EOF +{ + "blueprint": { + "customizations": { + "user": [ + { + "name": "${EDGE_USER}", + "password": "${EDGE_USER_PASSWORD}", + "key": "${SSH_KEY_PUB}", + "groups": [ + "wheel" + ] + } + ] + } + } +} +EOF +sudo rm -fr output && mkdir -p output +podman run \ + --rm \ + -it \ + --privileged \ + --pull=newer \ + --security-opt label=type:unconfined_t \ + -v $(pwd)/config.json:/config.json \ + -v $(pwd)/output:/output \ + -v /var/lib/containers/storage:/var/lib/containers/storage \ + ${BIB_URL} \ + --type qcow2 \ + --config /config.json \ + --rootfs xfs \ + quay.io/${QUAY_USERNAME}/greenboot-bootc:${TEST_UUID} + +########################################################### +## +## Provision vm with qcow2/iso artifacts +## +########################################################### +greenprint "Installing vm with bootc qcow2 image" +mv $(pwd)/output/qcow2/disk.qcow2 /var/lib/libvirt/images/${TEST_UUID}-disk.qcow2 +LIBVIRT_IMAGE_PATH_UEFI=/var/lib/libvirt/images/${TEST_UUID}-disk.qcow2 +sudo restorecon -Rv /var/lib/libvirt/images/ +sudo virt-install --name="${TEST_UUID}-uefi"\ + --disk path="${LIBVIRT_IMAGE_PATH_UEFI}",format=qcow2 \ + --ram 3072 \ + --vcpus 2 \ + --network network=integration,mac=34:49:22:B0:83:30 \ + --os-type linux \ + --os-variant ${OS_VARIANT} \ + --boot uefi \ + --nographics \ + --noautoconsole \ + --wait=-1 \ + --import \ + --noreboot +greenprint "Starting UEFI VM" +sudo virsh start "${TEST_UUID}-uefi" + +# Check for ssh ready to go. +greenprint "🛃 Checking for SSH is ready to go" +for _ in $(seq 0 30); do + RESULTS="$(wait_for_ssh_up $GUEST_ADDRESS)" + if [[ $RESULTS == 1 ]]; then + echo "SSH is ready now! 🥳" + break + fi + sleep 10 +done +check_result + +########################################################### +## +## Build upgrade container with failing-unit installed +## +########################################################### +greenprint "Building upgrade container" +tee Containerfile > /dev/null << EOF +FROM quay.io/${QUAY_USERNAME}/greenboot-bootc:${TEST_UUID} +RUN dnf install -y https://kite-webhook-prod.s3.amazonaws.com/greenboot-failing-unit-1.0-1.el8.noarch.rpm +EOF +podman build --retry=5 --retry-delay=10 -t quay.io/${QUAY_USERNAME}/greenboot-bootc:${TEST_UUID} -f Containerfile . +greenprint "Pushing upgrade container to quay.io" +podman push quay.io/${QUAY_USERNAME}/greenboot-bootc:${TEST_UUID} + +########################################################### +## +## Bootc upgrade and check if greenboot can rollback +## +########################################################### +greenprint "Bootc upgrade and reboot" +sudo ssh "${SSH_OPTIONS[@]}" -i "${SSH_KEY}" ${EDGE_USER}@${GUEST_ADDRESS} "echo ${EDGE_USER_PASSWORD} |sudo -S bootc upgrade" +sudo ssh "${SSH_OPTIONS[@]}" -i "${SSH_KEY}" ${EDGE_USER}@${GUEST_ADDRESS} "echo ${EDGE_USER_PASSWORD} |nohup sudo -S systemctl reboot &>/dev/null & exit" + +# Sleep 10 seconds here to make sure vm restarted already +sleep 180 + +# Check for ssh ready to go. +greenprint "🛃 Checking for SSH is ready to go" +for _ in $(seq 0 30); do + RESULTS="$(wait_for_ssh_up $GUEST_ADDRESS)" + if [[ $RESULTS == 1 ]]; then + echo "SSH is ready now! 🥳" + break + fi + sleep 10 +done +check_result + +# Add instance IP address into /etc/ansible/hosts +tee ${TEMPDIR}/inventory > /dev/null << EOF +[greenboot_guest] +${GUEST_ADDRESS} + +[greenboot_guest:vars] +ansible_python_interpreter=/usr/bin/python3 +ansible_user=${EDGE_USER} +ansible_private_key_file=${SSH_KEY} +ansible_ssh_common_args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" +ansible_become=yes +ansible_become_method=sudo +ansible_become_pass=${EDGE_USER_PASSWORD} +EOF + +# Test greenboot functionality +podman run --annotation run.oci.keep_original_groups=1 -v "$(pwd)":/work:z -v "${TEMPDIR}":/tmp:z \ + --rm quay.io/rhel-edge/ansible-runner:latest ansible-playbook -v -i /tmp/inventory greenboot-bootc.yaml || RESULTS=0 + +# Test result checking +check_result +exit 0 diff --git a/tests/greenboot-bootc.yaml b/tests/greenboot-bootc.yaml new file mode 100644 index 0000000..a15bbb4 --- /dev/null +++ b/tests/greenboot-bootc.yaml @@ -0,0 +1,120 @@ +--- +- hosts: greenboot_guest + become: no + vars: + total_counter: "0" + failed_counter: "0" + + + tasks: + # current target host's IP address + - debug: var=ansible_all_ipv4_addresses + - debug: var=ansible_facts['distribution_version'] + - debug: var=ansible_facts['distribution'] + - debug: var=ansible_facts['architecture'] + + - name: check bootc status + command: bootc status + ignore_errors: yes + + # case: check installed greenboot packages + - name: greenboot should be installed + block: + - name: greenboot should be installed + shell: rpm -qa | grep greenboot + register: result_greenboot_packages + + - assert: + that: + - "'greenboot-0' in result_greenboot_packages.stdout" + - "'greenboot-default-health-checks' in result_greenboot_packages.stdout" + fail_msg: "greenboot is not installed" + success_msg: "greenboot is installed" + always: + - set_fact: + total_counter: "{{ total_counter | int + 1 }}" + rescue: + - name: failed count + 1 + set_fact: + failed_counter: "{{ failed_counter | int + 1 }}" + + # case: check greenboot* services + - name: a list of greenboot* service should be enabled + block: + - name: a list of greenboot* service should be enabled + command: systemctl is-enabled greenboot-grub2-set-counter greenboot-grub2-set-success greenboot-healthcheck greenboot-rpm-ostree-grub2-check-fallback greenboot-status greenboot-task-runner redboot-auto-reboot redboot-task-runner + register: result_greenboot_service + + - assert: + that: + - result_greenboot_service.stdout == 'enabled\nenabled\nenabled\nenabled\ndisabled\ndisabled\nenabled\nenabled' + fail_msg: "Some of greenboot* services are not enabled" + success_msg: "All greenboot* services are enabled" + always: + - set_fact: + total_counter: "{{ total_counter | int + 1 }}" + rescue: + - name: failed count + 1 + set_fact: + failed_counter: "{{ failed_counter | int + 1 }}" + + # case: check greenboot fall back log + - name: fallback log should be found here + block: + - name: check boot-complete.target + command: systemctl --no-pager status boot-complete.target + become: yes + register: result + retries: 10 + delay: 60 + until: "'inactive' not in result.stdout" + + - name: fallback log should be found here + command: journalctl -b -0 -u greenboot -u greenboot-healthcheck -u greenboot-rpm-ostree-grub2-check-fallback -u greenboot-grub2-set-counter -u greenboot-grub2-set-success -u greenboot-status -u redboot -u redboot-auto-reboot -u redboot.target + become: yes + register: result_greenboot_log + + - assert: + that: + - "'FALLBACK BOOT DETECTED! Default bootc deployment has been rolled back' in result_greenboot_log.stdout" + - "'Script \\'00_required_scripts_start.sh\\' SUCCESS' in result_greenboot_log.stdout" + - "'Script \\'00_wanted_scripts_start.sh\\' SUCCESS' in result_greenboot_log.stdout" + - "'greenboot Health Checks Runner' in result_greenboot_log.stdout" + - "'Mark boot as successful in grubenv' in result_greenboot_log.stdout" + fail_msg: "Fallback log not found" + success_msg: "Found fallback log" + + always: + - set_fact: + total_counter: "{{ total_counter | int + 1 }}" + rescue: + - name: failed count + 1 + set_fact: + failed_counter: "{{ failed_counter | int + 1 }}" + + # case: check boot_success + - name: grubenv variables should contain boot_success=1 + block: + - name: grubenv variables should contain boot_success=1 + command: grub2-editenv list + register: result_grubenv + become: yes + + - assert: + that: + - "'boot_success=1' in result_grubenv.stdout" + fail_msg: "Not found boot_success=1" + success_msg: "Found boot_success=1" + always: + - set_fact: + total_counter: "{{ total_counter | int + 1 }}" + rescue: + - name: failed count + 1 + set_fact: + failed_counter: "{{ failed_counter | int + 1 }}" + + - assert: + that: + - failed_counter == "0" + fail_msg: "Run {{ total_counter }} tests, but {{ failed_counter }} of them failed" + success_msg: "Totally {{ total_counter }} test passed" diff --git a/tmt/plans/greenboot-test.fmf b/tmt/plans/greenboot-test.fmf new file mode 100644 index 0000000..fb6cd15 --- /dev/null +++ b/tmt/plans/greenboot-test.fmf @@ -0,0 +1,18 @@ +summary: Greenboot test plan +discover: + how: fmf + test: greenboot-test +execute: + how: tmt +provision: + hardware: + virtualization: + is-supported: true + cpu: + processors: ">= 2" + memory: ">= 6 GB" + +/greenboot-bootc-qcow2: + summary: Test greenboot with bootc qcow2 image + environment+: + TEST_CASE: bootc-qcow2 diff --git a/tmt/tests/greenboot-test.fmf b/tmt/tests/greenboot-test.fmf new file mode 100644 index 0000000..5f30ee1 --- /dev/null +++ b/tmt/tests/greenboot-test.fmf @@ -0,0 +1,2 @@ +test: ./test.sh +duration: 90m diff --git a/tmt/tests/test.sh b/tmt/tests/test.sh new file mode 100755 index 0000000..2223400 --- /dev/null +++ b/tmt/tests/test.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -euox pipefail + +cd ../../tests || exit 1 + +function run_tests() { + if [ "$TEST_CASE" = "bootc-qcow2" ]; then + ./greenboot-bootc-qcow2.sh + else + echo "Error: Test case $TEST_CASE not found!" + exit 1 + fi +} + +run_tests +exit 0