virt-s1 · henrywang · Apr 25, 2024 · Apr 23, 2024
diff --git a/.github/workflows/bib-image.yml b/.github/workflows/bib-image.yml
@@ -61,7 +61,7 @@ jobs:
     strategy:
       matrix:
         arch: [x86_64, aarch64]
-        image_type: [ami, qcow2, vmdk, raw, to-disk]
+        image_type: [ami, qcow2, vmdk, raw, anaconda-iso, to-disk]
         exclude:
           - arch: aarch64
             image_type: vmdk
@@ -76,6 +76,13 @@ jobs:
             platform: libvirt
           - image_type: to-disk
             platform: libvirt
+          - image_type: anaconda-iso
+            platform: libvirt
+          - image_type: anaconda-iso
+            arch: x86_64
+            firmware: bios
+          - image_type: anaconda-iso
+            firmware: uefi
     runs-on: ubuntu-latest
 
     steps:
@@ -99,7 +106,7 @@ jobs:
           tmt_plan_regex: "${{ matrix.image_type }}"
           tf_scope: private
           secrets: "TIER1_IMAGE_URL=${{ secrets.RHEL95_TIER1_IMAGE_URL }};QUAY_USERNAME=${{ secrets.QUAY_USERNAME }};QUAY_PASSWORD=${{ secrets.QUAY_PASSWORD }};QUAY_SECRET=${{ secrets.QUAY_SECRET }};DOWNLOAD_NODE=${{ secrets.DOWNLOAD_NODE }};AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }};AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }};GOVC_URL=${{ secrets.GOVC_URL }};GOVC_USERNAME=${{ secrets.GOVC_USERNAME }};GOVC_PASSWORD=${{ secrets.GOVC_PASSWORD }};RHC_AK=${{ secrets.RHC_AK }};RHC_ORGID=${{ secrets.RHC_ORGID }}"
-          variables: "PLATFORM=${{ matrix.platform }};ARCH=${{ matrix.arch }};IMAGE_TYPE=${{ matrix.image_type }};AWS_REGION=${{ secrets.AWS_REGION }};GOVC_INSECURE=1"
+          variables: "PLATFORM=${{ matrix.platform }};ARCH=${{ matrix.arch }};IMAGE_TYPE=${{ matrix.image_type }};AWS_REGION=${{ secrets.AWS_REGION }};GOVC_INSECURE=1;FIRMWARE=${{ matrix.firmware }}"
 
   rhel94-bib-image:
     needs: pr-info
@@ -110,7 +117,7 @@ jobs:
     strategy:
       matrix:
         arch: [x86_64, aarch64]
-        image_type: [ami, qcow2, vmdk, raw, to-disk]
+        image_type: [ami, qcow2, vmdk, raw, anaconda-iso, to-disk]
         exclude:
           - arch: aarch64
             image_type: vmdk
@@ -125,6 +132,13 @@ jobs:
             platform: libvirt
           - image_type: to-disk
             platform: libvirt
+          - image_type: anaconda-iso
+            platform: libvirt
+          - image_type: anaconda-iso
+            firmware: bios
+            arch: x86_64
+          - image_type: anaconda-iso
+            firmware: uefi
     runs-on: ubuntu-latest
 
     steps:
@@ -148,7 +162,7 @@ jobs:
           tmt_plan_regex: "${{ matrix.image_type }}"
           tf_scope: private
           secrets: "TIER1_IMAGE_URL=${{ secrets.RHEL94_TIER1_IMAGE_URL }};QUAY_USERNAME=${{ secrets.QUAY_USERNAME }};QUAY_PASSWORD=${{ secrets.QUAY_PASSWORD }};QUAY_SECRET=${{ secrets.QUAY_SECRET }};DOWNLOAD_NODE=${{ secrets.DOWNLOAD_NODE }};AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }};AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }};GOVC_URL=${{ secrets.GOVC_URL }};GOVC_USERNAME=${{ secrets.GOVC_USERNAME }};GOVC_PASSWORD=${{ secrets.GOVC_PASSWORD }};RHC_AK=${{ secrets.RHC_AK }};RHC_ORGID=${{ secrets.RHC_ORGID }}"
-          variables: "PLATFORM=${{ matrix.platform }};ARCH=${{ matrix.arch }};IMAGE_TYPE=${{ matrix.image_type }};AWS_REGION=${{ secrets.AWS_REGION }};GOVC_INSECURE=1"
+          variables: "PLATFORM=${{ matrix.platform }};ARCH=${{ matrix.arch }};IMAGE_TYPE=${{ matrix.image_type }};AWS_REGION=${{ secrets.AWS_REGION }};GOVC_INSECURE=1;FIRMWARE=${{ matrix.firmware }}"
 
   cs9-bib-image:
     needs: pr-info
@@ -159,7 +173,7 @@ jobs:
     strategy:
       matrix:
         arch: [x86_64, aarch64]
-        image_type: [ami, qcow2, vmdk, raw, to-disk]
+        image_type: [ami, qcow2, vmdk, raw, anaconda-iso, to-disk]
         exclude:
           - arch: aarch64
             image_type: vmdk
@@ -174,6 +188,13 @@ jobs:
             platform: libvirt
           - image_type: to-disk
             platform: libvirt
+          - image_type: anaconda-iso
+            platform: libvirt
+          - image_type: anaconda-iso
+            firmware: bios
+            arch: x86_64
+          - image_type: anaconda-iso
+            firmware: uefi
     runs-on: ubuntu-latest
 
     steps:
@@ -197,7 +218,7 @@ jobs:
           tmt_plan_regex: "${{ matrix.image_type }}"
           tf_scope: private
           secrets: "TIER1_IMAGE_URL=${{ secrets.CS9_TIER1_IMAGE_URL }};QUAY_USERNAME=${{ secrets.QUAY_USERNAME }};QUAY_PASSWORD=${{ secrets.QUAY_PASSWORD }};QUAY_SECRET=${{ secrets.QUAY_SECRET }};AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }};AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }};GOVC_URL=${{ secrets.GOVC_URL }};GOVC_USERNAME=${{ secrets.GOVC_USERNAME }};GOVC_PASSWORD=${{ secrets.GOVC_PASSWORD }}"
-          variables: "PLATFORM=${{ matrix.platform }};ARCH=${{ matrix.arch }};IMAGE_TYPE=${{ matrix.image_type }};AWS_REGION=${{ secrets.AWS_REGION }};GOVC_INSECURE=1"
+          variables: "PLATFORM=${{ matrix.platform }};ARCH=${{ matrix.arch }};IMAGE_TYPE=${{ matrix.image_type }};AWS_REGION=${{ secrets.AWS_REGION }};GOVC_INSECURE=1;FIRMWARE=${{ matrix.firmware }}"
 
   cs9-dev-bib-image:
     needs: pr-info
@@ -208,7 +229,7 @@ jobs:
     strategy:
       matrix:
         arch: [x86_64, aarch64]
-        image_type: [ami, qcow2, vmdk, raw, to-disk]
+        image_type: [ami, qcow2, vmdk, raw, anaconda-iso, to-disk]
         build_arch: [x86_64, aarch64]
         exclude:
           - image_type: vmdk
@@ -233,6 +254,12 @@ jobs:
           - image_type: to-disk
             arch: aarch64
             build_arch: x86_64
+          - image_type: anaconda-iso
+            arch: x86_64
+            build_arch: aarch64
+          - image_type: anaconda-iso
+            arch: aarch64
+            build_arch: x86_64
         include:
           - image_type: ami
             platform: aws
@@ -244,6 +271,13 @@ jobs:
             platform: libvirt
           - image_type: to-disk
             platform: libvirt
+          - image_type: anaconda-iso
+            platform: libvirt
+          - image_type: anaconda-iso
+            firmware: bios
+            arch: x86_64
+          - image_type: anaconda-iso
+            firmware: uefi
           - arch: x86_64
             build_arch: x86_64
             runner_compose: RHEL-9.5.0-Nightly
@@ -280,7 +314,7 @@ jobs:
           tmt_plan_regex: "${{ matrix.image_type }}"
           tf_scope: private
           secrets: "TIER1_IMAGE_URL=${{ secrets.CS9_DEV_TIER1_IMAGE_URL }};QUAY_USERNAME=${{ secrets.QUAY_USERNAME }};QUAY_PASSWORD=${{ secrets.QUAY_PASSWORD }};QUAY_SECRET=${{ secrets.QUAY_SECRET }};AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }};AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }};GOVC_URL=${{ secrets.GOVC_URL }};GOVC_USERNAME=${{ secrets.GOVC_USERNAME }};GOVC_PASSWORD=${{ secrets.GOVC_PASSWORD }}"
-          variables: "PLATFORM=${{ matrix.platform }};ARCH=${{ matrix.arch }};IMAGE_TYPE=${{ matrix.image_type }};AWS_REGION=${{ secrets.AWS_REGION }};GOVC_INSECURE=1"
+          variables: "PLATFORM=${{ matrix.platform }};ARCH=${{ matrix.arch }};IMAGE_TYPE=${{ matrix.image_type }};AWS_REGION=${{ secrets.AWS_REGION }};GOVC_INSECURE=1;FIRMWARE=${{ matrix.firmware }}"
 
   rhel9y-snapshot-bib-image:
     needs: pr-info
@@ -291,7 +325,7 @@ jobs:
     strategy:
       matrix:
         arch: [x86_64, aarch64]
-        image_type: [ami, qcow2, vmdk, raw, to-disk]
+        image_type: [ami, qcow2, vmdk, raw, anaconda-iso, to-disk]
         exclude:
           - arch: aarch64
             image_type: vmdk
@@ -306,6 +340,13 @@ jobs:
             platform: libvirt
           - image_type: to-disk
             platform: libvirt
+          - image_type: anaconda-iso
+            platform: libvirt
+          - image_type: anaconda-iso
+            firmware: bios
+            arch: x86_64
+          - image_type: anaconda-iso
+            firmware: uefi
     runs-on: ubuntu-latest
 
     steps:
@@ -329,4 +370,4 @@ jobs:
           tmt_plan_regex: "${{ matrix.image_type }}"
           tf_scope: private
           secrets: "TIER1_IMAGE_URL=${{ secrets.RHEL9Y_SNAPSHOT_IMAGE_URL }};QUAY_USERNAME=${{ secrets.QUAY_USERNAME }};QUAY_PASSWORD=${{ secrets.QUAY_PASSWORD }};QUAY_SECRET=${{ secrets.QUAY_SECRET }};DOWNLOAD_NODE=${{ secrets.DOWNLOAD_NODE }};AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }};AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }};GOVC_URL=${{ secrets.GOVC_URL }};GOVC_USERNAME=${{ secrets.GOVC_USERNAME }};GOVC_PASSWORD=${{ secrets.GOVC_PASSWORD }};RHC_AK=${{ secrets.RHC_AK }};RHC_ORGID=${{ secrets.RHC_ORGID }}"
-          variables: "PLATFORM=${{ matrix.platform }};ARCH=${{ matrix.arch }};IMAGE_TYPE=${{ matrix.image_type }};AWS_REGION=${{ secrets.AWS_REGION }};GOVC_INSECURE=1"
+          variables: "PLATFORM=${{ matrix.platform }};ARCH=${{ matrix.arch }};IMAGE_TYPE=${{ matrix.image_type }};AWS_REGION=${{ secrets.AWS_REGION }};GOVC_INSECURE=1;FIRMWARE=${{ matrix.firmware }}"
diff --git a/bib-image.sh b/bib-image.sh
@@ -28,6 +28,9 @@ INVENTORY_FILE="${TEMPDIR}/inventory"
 
 REPLACE_CLOUD_USER=""
 
+# For anaconda-iso test
+FIRMWARE="${FIRMWARE:-bios}"
+
 greenprint "Login quay.io"
 sudo podman login -u "${QUAY_USERNAME}" -p "${QUAY_PASSWORD}" quay.io
 
@@ -59,6 +62,7 @@ yum_repos:
     gpgcheck: false
 EOF
         GUEST_ID_DC70="rhel9_64Guest"
+        BOOT_ARGS="uefi"
         ;;
     "centos")
         # work with old TEST_OS variable value centos-stream-9
@@ -71,11 +75,13 @@ EOF
             REPLACE_CLOUD_USER='RUN sed -i "s/name: cloud-user/name: ec2-user/g" /etc/cloud/cloud.cfg'
         fi
         GUEST_ID_DC70="centos9_64Guest"
+        BOOT_ARGS="uefi,firmware.feature0.name=secure-boot,firmware.feature0.enabled=no"
         ;;
     "fedora")
         SSH_USER="fedora"
         ADD_REPO=""
         ADD_RHC=""
+        BOOT_ARGS="uefi"
         ;;
     *)
         redprint "Variable TIER1_IMAGE_URL is not supported"
@@ -101,9 +107,13 @@ case "$ARCH" in
         ;;
 esac
 
-
 greenprint "Create ${TEST_OS} installation Containerfile"
 sed -i "s|^FROM.*|FROM $TIER1_IMAGE_URL\n$ADD_REPO\n$ADD_RHC|" "$INSTALL_CONTAINERFILE"
+
+if [[ "$LAYERED_IMAGE" == "useradd-ssh" ]]; then
+   sed -i "s|exampleuser|$SSH_USER|g" "$INSTALL_CONTAINERFILE"
+fi
+
 tee -a "$INSTALL_CONTAINERFILE" > /dev/null << EOF
 RUN dnf -y clean all
 $REPLACE_CLOUD_USER
@@ -118,7 +128,11 @@ greenprint "Check $TEST_OS installation Containerfile"
 cat "$INSTALL_CONTAINERFILE"
 
 greenprint "Build $TEST_OS installation container image"
-sudo podman build --platform "$BUILD_PLATFORM" --tls-verify=false --retry=5 --retry-delay=10 -t "${TEST_IMAGE_NAME}:${QUAY_REPO_TAG}" "$LAYERED_DIR"
+if [[ "$LAYERED_IMAGE" == "useradd-ssh" ]]; then
+    sudo podman build --platform "$BUILD_PLATFORM" --tls-verify=false --retry=5 --retry-delay=10 --build-arg "sshpubkey=$(cat "${SSH_KEY_PUB}")" -t "${TEST_IMAGE_NAME}:${QUAY_REPO_TAG}" "$LAYERED_DIR"
+else
+    sudo podman build --platform "$BUILD_PLATFORM" --tls-verify=false --retry=5 --retry-delay=10 -t "${TEST_IMAGE_NAME}:${QUAY_REPO_TAG}" "$LAYERED_DIR"
+fi
 
 [[ $- =~ x ]] && debug=1 && set +x
 sed "s/REPLACE_ME/${QUAY_SECRET}/g" files/auth.template | tee auth.json > /dev/null
@@ -358,6 +372,42 @@ EOF
         greenprint "Clean up userdata.yaml and metadata.yaml"
         rm -f userdata.yaml metadata.yaml
         ;;
+    "anaconda-iso")
+        greenprint "Build $TEST_OS $IMAGE_TYPE image"
+        mkdir -p output
+        sudo podman run \
+            --rm \
+            -it \
+            --privileged \
+            --pull=newer \
+            --tls-verify=false \
+            --security-opt label=type:unconfined_t \
+            -v "$(pwd)/output":/output \
+            -v /var/lib/containers/storage:/var/lib/containers/storage \
+            "$BIB_IMAGE_URL" \
+            --type "$IMAGE_TYPE" \
+            --target-arch "$ARCH" \
+            --chown "$(id -u "$(whoami)"):$(id -g "$(whoami)")" \
+            --local \
+            "$LOCAL_IMAGE_URL"
+
+        sudo mv output/bootiso/install.iso /var/lib/libvirt/images && sudo rm -rf output
+
+        greenprint "💾 Create vm qcow2 files for ISO installation"
+        sudo qemu-img create -f qcow2 "/var/lib/libvirt/images/disk.qcow2" 10G
+
+        greenprint "Deploy $IMAGE_TYPE instance"
+        ansible-playbook -v \
+            -i "$INVENTORY_FILE" \
+            -e test_os="$TEST_OS" \
+            -e ssh_key_pub="$SSH_KEY_PUB" \
+            -e ssh_user="$SSH_USER" \
+            -e inventory_file="$INVENTORY_FILE" \
+            -e bib="true" \
+            -e firmware="$FIRMWARE" \
+            -e boot_args="$BOOT_ARGS" \
+            "playbooks/deploy-libvirt.yaml"
+        ;;
     "to-disk")
         greenprint "💾 Create disk.raw"
         sudo truncate -s 10G disk.raw

diff --git a/playbooks/deploy-libvirt.yaml b/playbooks/deploy-libvirt.yaml
@@ -12,6 +12,8 @@
     instance_name: "bootc-{{ test_os }}"
     image_path: "/var/lib/libvirt/images"
     bib: "false"
+    firmware: ""
+    boot_args: ""
     os_variant:
       centos-stream-9: centos-stream9
       rhel-9-5: rhel9-unknown
@@ -89,6 +91,7 @@
       loop:
         - user-data
         - meta-data
+      when: firmware == ""
 
     # virt-install with --cloud-init always shutdown vm on the first reboot
     # https://github.com/virt-manager/virt-manager/issues/497

diff --git a/playbooks/remove.yaml b/playbooks/remove.yaml
@@ -80,17 +80,21 @@
         - name: "Destroy vm"
           command: virsh destroy {{ instance_name }}
           become: true
-        - name: "Undefine vm on x86_64"
+          ignore_errors: true
+        - name: "Undefine vm"
           command: virsh undefine {{ instance_name }}
           become: true
-          when: ansible_facts['architecture'] == "x86_64"
-        - name: "Undefine vm on aarch64"
+          register: result_undefine
+          ignore_errors: true
+        - name: "Undefine vm with --nvram"
           command: virsh undefine {{ instance_name }} --nvram
           become: true
-          when: ansible_facts['architecture'] == "aarch64"
+          ignore_errors: true
+          when: result_undefine is failed
         - name: "Delete disk file"
           command: virsh vol-delete --pool images "{{ rhel_guest_image_fname }}"
           become: true
+          ignore_errors: true
       when: platform == "libvirt"
 
     - name: Cancel beaker job

diff --git a/playbooks/templates/virt-install.bash.j2 b/playbooks/templates/virt-install.bash.j2
@@ -1,18 +1,29 @@
 #!/bin/bash
 
 virt-install \
+{% if firmware == '' %}
     --import \
+{% endif %}
     --name {{ instance_name }} \
     --ram 3072 \
     --vcpus 2 \
     --os-variant {{ os_variant[test_os] }} \
     --network default \
     --disk size=10,path="{{ image_path }}/{{ rhel_guest_image_fname }}" \
+{% if firmware == 'bios' %}
+    --cdrom "{{ image_path }}/install.iso" \
+    --noreboot \
+{% elif firmware == 'uefi' %}
+    --cdrom "{{ image_path }}/install.iso" \
+    --noreboot \
+    --boot {{ boot_args }} \
+{% endif %}
+{% if firmware == '' %}
     --cloud-init user-data="{{ playbook_dir }}/libvirt-user-data",meta-data="{{ playbook_dir }}/libvirt-meta-data",disable=on \
+{% endif %}
 {% if bib == 'false' %}
     --filesystem={{ air_gapped_dir }},mount_tag,driver.type=virtiofs,accessmode=passthrough \
     --memorybacking=source.type=memfd,access.mode=shared \
 {% endif %}
-    --graphics none \
     --noautoconsole \
     --wait
diff --git a/tmt/plans/bib-image.fmf b/tmt/plans/bib-image.fmf
@@ -95,21 +95,20 @@ execute:
     - how: shell
       script: curl -L -o - "https://github.com/vmware/govmomi/releases/latest/download/govc_$(uname -s)_$(uname -m).tar.gz" | sudo tar -C /usr/local/bin -xvzf - govc
 
-/iso:
+/anaconda-iso:
   summary: Use bib generate ISO image and test locally (nested)
-  tag: libvirt
+  tag: /anaconda-iso
   environment+:
     PLATFORM: libvirt
-    IMAGE_TYPE: iso
-    LAYERED_IMAGE: cloud-init
+    IMAGE_TYPE: anaconda-iso
+    LAYERED_IMAGE: useradd-ssh
   prepare+:
     - how: install
       package:
-        - python3-libvirt
-        - python3-lxml
+        - qemu-img
         - qemu-kvm
         - libvirt
-        - xorriso
+        - virt-install
   adjust+:
     - when: arch == ppc64le
       enabled: false