Replace Ubuntu -> Debian SID

After years of trying, use of Ubuntu VMs in CI has proven too difficult
to maintain along with an ever-updating set of dependencies.  This is
different with Debian SID, since it more/less pulls packages directly
from their upstream sources.  Switch to using Debian SID for all
containers-org CI.

Signed-off-by: Chris Evich <[email protected]>

.cirrus.yml

@@ -85,14 +85,16 @@ container_images_task: &container_images
           env:
             TARGET_NAME: 'prior-fedora_podman'
             DEST_FQIN: *fqin
-        - name: *name
-          env:
-            TARGET_NAME: 'ubuntu_podman'
-            DEST_FQIN: *fqin
         - name: *name
           env:
             TARGET_NAME: 'skopeo_cidev'
             DEST_FQIN: *fqin
+        # Upgrading from stable -> SID fails for container-images
+        # TODO: Re-enable once fixed
+        # - name: *name
+        #   env:
+        #     TARGET_NAME: 'debian'
+        #     DEST_FQIN: *fqin
     env: &image_env
         # For quay.io/libpod namespace
         REG_USERNAME: ENCRYPTED[de755aef351c501ee480231c24eae25b15e2b2a2b7c629f477c1d427fc5269e360bb358a53bd8914605bae588e99b52a]
@@ -163,7 +165,7 @@ base_images_task:
             PACKER_BUILDS: "fedora-aws-arm64"
         - <<: *base_image
           env:
-            PACKER_BUILDS: "ubuntu"
+            PACKER_BUILDS: "debian"
     env:
       GAC_JSON: &gac_json ENCRYPTED[7fba7fb26ab568ae39f799ab58a476123206576b0135b3d1019117c6d682391370c801e149f29324ff4b50133012aed9]
       AWS_INI: &aws_ini ENCRYPTED[4cd69097cd29a9899e51acf3bbacceeb83cb5c907d272ca1e2a8ccd515b03f2368a0680870c0d120fc32bc578bb0a930]
@@ -219,7 +221,7 @@ cache_images_task:
             PACKER_BUILDS: "build-push"
         - <<: *cache_image
           env:
-            PACKER_BUILDS: "ubuntu"
+            PACKER_BUILDS: "debian"
     env:
       GAC_JSON: *gac_json
       AWS_INI: *aws_ini
@@ -283,12 +285,12 @@ test_imgts_task: &imgts
             image-builder-${IMG_SFX}
             fedora-b${IMG_SFX}
             prior-fedora-b${IMG_SFX}
-            ubuntu-b${IMG_SFX}
+            debian-b${IMG_SFX}
             fedora-c${IMG_SFX}
             prior-fedora-c${IMG_SFX}
             fedora-netavark-c${IMG_SFX}
             fedora-podman-py-c${IMG_SFX}
-            ubuntu-c${IMG_SFX}
+            debian-c${IMG_SFX}
             build-push-c${IMG_SFX}
         EC2IMGNAMES: |
             fedora-aws-b${IMG_SFX}

Makefile

@@ -23,8 +23,10 @@ export PRIOR_FEDORA_RELEASE = 36
 # See import_images/README.md
 export FEDORA_IMPORT_IMG_SFX = 1669819494
 
-export UBUNTU_RELEASE = 22.04
-export UBUNTU_BASE_FAMILY = ubuntu-2204-lts
+# Automation assumes the actual release number (after SID upgrade)
+# is always one-greater than the latest DEBIAN_BASE_FAMILY (GCE image).
+export DEBIAN_RELEASE = 12
+export DEBIAN_BASE_FAMILY = debian-11
 
 IMPORT_FORMAT = vhdx
 
@@ -379,10 +381,6 @@ fedora_podman:  ## Build Fedora podman development container
 prior-fedora_podman:  ## Build Prior-Fedora podman development container
 	$(call build_podman_container,$@,$(PRIOR_FEDORA_RELEASE))
 
-.PHONY: ubuntu_podman
-ubuntu_podman:  ## Build Ubuntu podman development container
-	$(call build_podman_container,$@,$(UBUNTU_RELEASE))
-
 $(_TEMPDIR)/%_podman.tar: podman/Containerfile podman/setup.sh $(wildcard base_images/*.sh) $(wildcard cache_images/*.sh) $(_TEMPDIR)/.cache/%
 	podman build -t $*_podman:$(call err_if_empty,_IMG_SFX) \
 		--security-opt seccomp=unconfined \

README.md

@@ -73,10 +73,10 @@ please [see it's documentation page](https://www.packer.io/docs).
    Use this VM to
    [build and then import base-level VM
    image](README.md#the-base-images-overview-step-3) for supported platforms
-   (Fedora or Ubuntu; as of this writing).  For GCE use, convert the
+   (Fedora or Debian; as of this writing).  For GCE use, convert the
    generic distribution provided QCOW files, into bootable *GCE VMs*.  For
    AWS, boot the pre-build AMI's, add minimal tooling, and save them as
-   private, non-expiring AMIs.  In parallel, build Fedora and Ubuntu
+   private, non-expiring AMIs.  In parallel, build Fedora and Debian
    container images and push them to ``quay.io/libpod/<name>_podman``
 
 4. [Boot *VMs* from each image produced in step
@@ -126,7 +126,7 @@ see step 4 below.
 * The bulk of the packaging work occurs next, from the `cache_images/*_packaging.sh`
   scripts.  **This is most likely what you want to modify.**
 
-* Unlike the Fedora and Ubuntu scripts, the `build-push` VM image is not
+* Unlike the Fedora and Debian scripts, the `build-push` VM image is not
   for general-purpose use.  It's intended to be used by it's embedded
   `main.sh` script, in downstream repositories for building container images.
   The image and `main.sh` are both tightly coupled with `build-push` tool
@@ -258,13 +258,13 @@ an overview of the process followed **by automation** to produce the
 ### Podman
 
 Several instances of the image-builder VM are used to create container
-images.  In particular, Fedora and Ubuntu images are created that
+images.  In particular, Fedora and Debian images are created that
 more-or-less duplicate the setup of the VM Cache-images.  They are
 then automatically pushed to:
 
 * https://quay.io/repository/libpod/fedora_podman
 * https://quay.io/repository/libpod/prior-fedora_podman
-* https://quay.io/repository/libpod/ubuntu_podman
+* https://quay.io/repository/libpod/debian_podman
 
 The meaning of *prior* and *current*, is defined by the contents of
 the `*_release` files within the `podman` subdirectory.  This is
@@ -419,7 +419,7 @@ producing images proceeds as follows:
 3. If you wish to build only a subset of available images, list the names
    you want as comma-separated values of the `PACKER_BUILDS` variable.  Be
    sure you *export* this variable so that `make` has access to it.  For
-   example, `export PACKER_BUILDS=ubuntu,prior-fedora`.
+   example, `export PACKER_BUILDS=debian,prior-fedora`.
 
 4. Still within the container, again ensure you have plenty of network and CPU
    resources available.  Build the VM Base images by executing the command

base_images/cloud-init/debian/cloud.cfg.d/40_defuser.cfg

@@ -0,0 +1 @@
+../../fedora/cloud.cfg.d/40_defuser.cfg

base_images/cloud-init/debian/cloud.cfg.d/40_enable_root.cfg

@@ -0,0 +1 @@
+../../fedora/cloud.cfg.d/40_enable_root.cfg

base_images/cloud-init/fedora/cloud.cfg.d/40_defuser.cfg

@@ -0,0 +1,2 @@
+# Avoid adding any users to the system by default.  GCP OSLogin will do it at runtime.
+users: {}

base_images/cloud-init/fedora/cloud.cfg.d/40_enable_root.cfg

@@ -0,0 +1 @@
+disable_root: 0

base_images/cloud.yml

@@ -28,19 +28,19 @@ variables:  # Empty value means it must be passed in on command-line
 
     FEDORA_IMPORT_IMG_SFX: "{{env `FEDORA_IMPORT_IMG_SFX`}}"
 
-    UBUNTU_RELEASE: "{{env `UBUNTU_RELEASE`}}"
-    UBUNTU_BASE_FAMILY: "{{env `UBUNTU_BASE_FAMILY`}}"
+    DEBIAN_RELEASE: "{{env `DEBIAN_RELEASE`}}"
+    DEBIAN_BASE_FAMILY: "{{env `DEBIAN_BASE_FAMILY`}}"
 
 
 builders:
-    - name: 'ubuntu'
+    - name: 'debian'
       type: 'googlecompute'
       # Prefix IMG_SFX with "b" so this is never confused with a cache_image name
       image_name: '{{build_name}}-b{{user `IMG_SFX`}}'
       image_family: '{{build_name}}-base'
       image_description: 'Built in https://cirrus-ci.com/task/{{user `CIRRUS_TASK_ID`}}'
-      source_image_family: '{{user `UBUNTU_BASE_FAMILY`}}'
-      source_image_project_id: 'ubuntu-os-cloud'
+      source_image_family: '{{user `DEBIAN_BASE_FAMILY`}}'
+      source_image_project_id: 'debian-cloud'
       project_id: '{{user `GCP_PROJECT_ID`}}'
       # Can't use env. var for this, googlecompute-import only supports filepath
       account_file: '{{user `GAC_FILEPATH`}}'
@@ -49,10 +49,10 @@ builders:
       # Identify the instance
       labels: &imgcpylabels
         sfx: '{{user `IMG_SFX`}}'
-        src: '{{user `UBUNTU_BASE_FAMILY`}}'
+        src: '{{user `DEBIAN_BASE_FAMILY`}}'
         stage: 'base'
         arch: 'x86_64'
-        release: 'ubuntu-{{user `UBUNTU_RELEASE` | clean_resource_name}}'
+        release: 'debian-{{user `DEBIAN_RELEASE` | clean_resource_name}}'
       # Gotcha: https://www.packer.io/docs/builders/googlecompute#gotchas
       ssh_username: 'packer'
       temporary_key_pair_type: ed25519
@@ -183,7 +183,7 @@ builders:
       snapshot_tags: *awsarm64tags
 
 
-provisioners:  # Ubuntu images come bundled with GCE integrations provisioned
+provisioners:  # Debian images come bundled with GCE integrations provisioned
     - type: 'shell'
       inline:
         - 'set -e'
@@ -193,17 +193,17 @@ provisioners:  # Ubuntu images come bundled with GCE integrations provisioned
       source: '{{ pwd }}/'
       destination: '/tmp/automation_images/'
 
-    - except: ['ubuntu']
+    - except: ['debian']
       type: 'shell'
       inline:
         - 'set -e'
         - '/bin/bash /tmp/automation_images/base_images/fedora_base-setup.sh'
 
-    - only: ['ubuntu']
+    - only: ['debian']
       type: 'shell'
       inline:
         - 'set -e'
-        - '/bin/bash /tmp/automation_images/base_images/ubuntu_base-setup.sh'
+        - '/bin/bash /tmp/automation_images/base_images/debian_base-setup.sh'
 
 post-processors:
     # Must be double-nested to guarantee execution order

base_images/debian_base-setup.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+# This script is intended to be run by packer, inside an Debian VM.
+# It's purpose is to configure the VM for importing into google cloud,
+# so that it will boot in GCE and be accessable for further use.
+
+set -eo pipefail
+
+SCRIPT_FILEPATH=$(realpath "${BASH_SOURCE[0]}")
+SCRIPT_DIRPATH=$(dirname "$SCRIPT_FILEPATH")
+REPO_DIRPATH=$(realpath "$SCRIPT_DIRPATH/../")
+
+# Run as quickly as possible after boot
+/bin/bash $REPO_DIRPATH/systemd_banish.sh
+
+# shellcheck source=./lib.sh
+source "$REPO_DIRPATH/lib.sh"
+
+# Switch to Debian Unstable (SID)
+cat << EOF | sudo tee /etc/apt/sources.list
+deb http://deb.debian.org/debian/ unstable main
+deb-src http://deb.debian.org/debian/ unstable main
+EOF
+
+declare -a PKGS
+PKGS=( \
+    coreutils
+    curl
+    cloud-init
+    gawk
+    git
+    openssh-client
+    openssh-server
+    rng-tools5
+    software-properties-common
+)
+
+echo "Updating package source lists"
+$SUDO apt-get -qq -y update
+echo "Upgrading to SID"
+$SUDO apt-get -qq -y full-upgrade
+echo "Installing basic, necessary packages."
+$SUDO apt-get -qq -y install "${PKGS[@]}"
+
+# compatibility / usefullness of all automated scripting (which is bash-centric)
+$SUDO DEBCONF_DB_OVERRIDE='File{'$SCRIPT_DIRPATH/no_dash.dat'}' \
+    dpkg-reconfigure dash
+
+# Ref: https://wiki.debian.org/DebianReleases
+# CI automation needs a *sortable* OS version/release number to select/perform/apply
+# runtime configuration and workarounds.  Since switching to Unstable/SID, a
+# numeric release version is not available. While an imperfect solution,
+# base an artificial version off the 'base-files' package version, right-padded with
+# zeros to ensure sortability (i.e. "12.02" < "12.13").
+base_files_version=$(dpkg -s base-files | awk '/Version:/{print $2}')
+base_major=$(cut -d. -f 1 <<<"$base_files_version")
+base_minor=$(cut -d. -f 2 <<<"$base_files_version")
+sortable_version=$(printf "%02d.%02d" $base_major $base_minor)
+echo "WARN: This is NOT an official version number.  It's for CI-automation purposes only."
+echo "VERSION_ID=\"$sortable_version\"" | \
+    $SUDO tee -a /etc/os-release
+
+install_automation_tooling
+
+if ! ((CONTAINER)); then
+    custom_cloud_init
+    $SUDO systemctl enable rngd
+fi
+
+finalize

cache_images/cloud.yml

@@ -19,11 +19,11 @@ variables:  # Empty value means it must be passed in on command-line
     # See Makefile for definitions
     FEDORA_RELEASE: "{{env `FEDORA_RELEASE`}}"
     PRIOR_FEDORA_RELEASE: "{{env `PRIOR_FEDORA_RELEASE`}}"
-    UBUNTU_RELEASE: "{{env `UBUNTU_RELEASE`}}"
+    DEBIAN_RELEASE: "{{env `DEBIAN_RELEASE`}}"
 
 builders:
     - &gce_hosted_image
-      name: 'ubuntu'
+      name: 'debian'
       type: 'googlecompute'
       # N/B: This implies base images always built with same IMG_SFX.
       source_image: '{{ build_name }}-b{{user `IMG_SFX`}}'
@@ -39,7 +39,7 @@ builders:
       labels: &gce_labels # For the VM
         sfx: '{{user `IMG_SFX`}}'
         src: '{{ build_name }}-b{{user `IMG_SFX` }}'
-        release: 'ubuntu-{{user `UBUNTU_RELEASE` | clean_resource_name}}'
+        release: 'debian-{{user `DEBIAN_RELEASE` | clean_resource_name}}'
         stage: cache
       ssh_username: packer  # arbitrary, packer will create & setup w/ temp. keypair
       ssh_pty: 'true'
@@ -179,17 +179,17 @@ provisioners:
       source: '{{ pwd }}/'
       destination: "/tmp/automation_images"
 
-    - except: ['ubuntu']
+    - except: ['debian']
       type: 'shell'
       inline:
         - 'set -e'
         - '/bin/bash /tmp/automation_images/cache_images/fedora_setup.sh'
 
-    - only: ['ubuntu']
+    - only: ['debian']
       type: 'shell'
       inline:
         - 'set -e'
-        - '/bin/bash /tmp/automation_images/cache_images/ubuntu_setup.sh'
+        - '/bin/bash /tmp/automation_images/cache_images/debian_setup.sh'
 
 post-processors:
     # This is critical for human-interaction.  Copntents will be used