Replace Ubuntu -> Debian SID

After years of trying, use of Ubuntu VMs in CI has proven too difficult to maintain along with an ever-updating set of dependencies. This is different with Debian SID, since it more/less pulls packages directly from their upstream sources. Switch to using Debian SID for all containers-org CI. Signed-off-by: Chris Evich <[email protected]>
containers · Jan 19, 2023 · 3566788 · 3566788
1 parent 75ad2e8
commit 3566788
Show file tree

Hide file tree

Showing 14 changed files with 65 additions and 200 deletions.
diff --git a/.cirrus.yml b/.cirrus.yml
@@ -90,10 +90,6 @@ container_images_task: &container_images
           env:
             TARGET_NAME: 'prior-fedora_podman'
             DEST_FQIN: *fqin
-        - name: *name
-          env:
-            TARGET_NAME: 'ubuntu_podman'
-            DEST_FQIN: *fqin
         - name: *name
           env:
             TARGET_NAME: 'skopeo_cidev'
@@ -168,7 +164,7 @@ base_images_task:
             PACKER_BUILDS: "fedora-aws-arm64"
         - <<: *base_image
           env:
-            PACKER_BUILDS: "ubuntu"
+            PACKER_BUILDS: "debian"
     env:
       GAC_JSON: &gac_json ENCRYPTED[7fba7fb26ab568ae39f799ab58a476123206576b0135b3d1019117c6d682391370c801e149f29324ff4b50133012aed9]
       AWS_INI: &aws_ini ENCRYPTED[4cd69097cd29a9899e51acf3bbacceeb83cb5c907d272ca1e2a8ccd515b03f2368a0680870c0d120fc32bc578bb0a930]
@@ -224,7 +220,7 @@ cache_images_task:
             PACKER_BUILDS: "build-push"
         - <<: *cache_image
           env:
-            PACKER_BUILDS: "ubuntu"
+            PACKER_BUILDS: "debian"
     env:
       GAC_JSON: *gac_json
       AWS_INI: *aws_ini
@@ -288,12 +284,12 @@ test_imgts_task: &imgts
             image-builder-${IMG_SFX}
             fedora-b${IMG_SFX}
             prior-fedora-b${IMG_SFX}
-            ubuntu-b${IMG_SFX}
+            debian-b${IMG_SFX}
             fedora-c${IMG_SFX}
             prior-fedora-c${IMG_SFX}
             fedora-netavark-c${IMG_SFX}
             fedora-podman-py-c${IMG_SFX}
-            ubuntu-c${IMG_SFX}
+            debian-c${IMG_SFX}
             build-push-c${IMG_SFX}
         EC2IMGNAMES: |
             fedora-aws-b${IMG_SFX}

diff --git a/Makefile b/Makefile
@@ -23,8 +23,8 @@ export PRIOR_FEDORA_RELEASE = 36
 # See import_images/README.md
 export FEDORA_IMPORT_IMG_SFX = 1669819494
 
-export UBUNTU_RELEASE = 22.04
-export UBUNTU_BASE_FAMILY = ubuntu-2204-lts
+export DEBIAN_RELEASE = 11
+export DEBIAN_BASE_FAMILY = debian-11
 
 IMPORT_FORMAT = vhdx
 
@@ -372,10 +372,6 @@ fedora_podman:  ## Build Fedora podman development container
 prior-fedora_podman:  ## Build Prior-Fedora podman development container
 	$(call build_podman_container,$@,$(PRIOR_FEDORA_RELEASE))
 
-.PHONY: ubuntu_podman
-ubuntu_podman:  ## Build Ubuntu podman development container
-	$(call build_podman_container,$@,$(UBUNTU_RELEASE))
-
 $(_TEMPDIR)/%_podman.tar: podman/Containerfile podman/setup.sh $(wildcard base_images/*.sh) $(wildcard cache_images/*.sh) $(_TEMPDIR)/.cache/%
 	podman build -t $*_podman:$(call err_if_empty,IMG_SFX) \
 		--security-opt seccomp=unconfined \

diff --git a/README.md b/README.md
@@ -73,10 +73,10 @@ please [see it's documentation page](https://www.packer.io/docs).
    Use this VM to
    [build and then import base-level VM
    image](README.md#the-base-images-overview-step-3) for supported platforms
-   (Fedora or Ubuntu; as of this writing).  For GCE use, convert the
+   (Fedora or Debian; as of this writing).  For GCE use, convert the
    generic distribution provided QCOW files, into bootable *GCE VMs*.  For
    AWS, boot the pre-build AMI's, add minimal tooling, and save them as
-   private, non-expiring AMIs.  In parallel, build Fedora and Ubuntu
+   private, non-expiring AMIs.  In parallel, build Fedora and Debian
    container images and push them to ``quay.io/libpod/<name>_podman``
 
 4. [Boot *VMs* from each image produced in step
@@ -126,7 +126,7 @@ see step 4 below.
 * The bulk of the packaging work occurs next, from the `cache_images/*_packaging.sh`
   scripts.  **This is most likely what you want to modify.**
 
-* Unlike the Fedora and Ubuntu scripts, the `build-push` VM image is not
+* Unlike the Fedora and Debian scripts, the `build-push` VM image is not
   for general-purpose use.  It's intended to be used by it's embedded
   `main.sh` script, in downstream repositories for building container images.
   The image and `main.sh` are both tightly coupled with `build-push` tool
@@ -259,13 +259,13 @@ an overview of the process followed **by automation** to produce the
 ### Podman
 
 Several instances of the image-builder VM are used to create container
-images.  In particular, Fedora and Ubuntu images are created that
+images.  In particular, Fedora and Debian images are created that
 more-or-less duplicate the setup of the VM Cache-images.  They are
 then automatically pushed to:
 
 * https://quay.io/repository/libpod/fedora_podman
 * https://quay.io/repository/libpod/prior-fedora_podman
-* https://quay.io/repository/libpod/ubuntu_podman
+* https://quay.io/repository/libpod/debian_podman
 
 The meaning of *prior* and *current*, is defined by the contents of
 the `*_release` files within the `podman` subdirectory.  This is
@@ -426,7 +426,7 @@ producing images proceeds as follows:
 4. If you wish to build only a subset of available images, list the names
    you want as comma-separated values of the `PACKER_BUILDS` variable.  Be
    sure you *export* this variable so that `make` has access to it.  For
-   example, `export PACKER_BUILDS=ubuntu,prior-fedora`.
+   example, `export PACKER_BUILDS=debian,prior-fedora`.
 
 4. Still within the container, again ensure you have plenty of network and CPU
    resources available.  Build the VM Base images by executing the command

diff --git a/base_images/cloud.yml b/base_images/cloud.yml
@@ -28,19 +28,19 @@ variables:  # Empty value means it must be passed in on command-line
 
     FEDORA_IMPORT_IMG_SFX: "{{env `FEDORA_IMPORT_IMG_SFX`}}"
 
-    UBUNTU_RELEASE: "{{env `UBUNTU_RELEASE`}}"
-    UBUNTU_BASE_FAMILY: "{{env `UBUNTU_BASE_FAMILY`}}"
+    DEBIAN_RELEASE: "{{env `DEBIAN_RELEASE`}}"
+    DEBIAN_BASE_FAMILY: "{{env `DEBIAN_BASE_FAMILY`}}"
 
 
 builders:
-    - name: 'ubuntu'
+    - name: 'debian'
       type: 'googlecompute'
       # Prefix IMG_SFX with "b" so this is never confused with a cache_image name
       image_name: '{{build_name}}-b{{user `IMG_SFX`}}'
       image_family: '{{build_name}}-base'
       image_description: 'Built in https://cirrus-ci.com/task/{{user `CIRRUS_TASK_ID`}}'
-      source_image_family: '{{user `UBUNTU_BASE_FAMILY`}}'
-      source_image_project_id: 'ubuntu-os-cloud'
+      source_image_family: '{{user `DEBIAN_BASE_FAMILY`}}'
+      source_image_project_id: 'debian-cloud'
       project_id: '{{user `GCP_PROJECT_ID`}}'
       # Can't use env. var for this, googlecompute-import only supports filepath
       account_file: '{{user `GAC_FILEPATH`}}'
@@ -49,10 +49,10 @@ builders:
       # Identify the instance
       labels: &imgcpylabels
         sfx: '{{user `IMG_SFX`}}'
-        src: '{{user `UBUNTU_BASE_FAMILY`}}'
+        src: '{{user `DEBIAN_BASE_FAMILY`}}'
         stage: 'base'
         arch: 'x86_64'
-        release: 'ubuntu-{{user `UBUNTU_RELEASE` | clean_resource_name}}'
+        release: 'debian-{{user `DEBIAN_RELEASE` | clean_resource_name}}'
       # Gotcha: https://www.packer.io/docs/builders/googlecompute#gotchas
       ssh_username: 'packer'
       temporary_key_pair_type: ed25519
@@ -183,7 +183,7 @@ builders:
       snapshot_tags: *awsarm64tags
 
 
-provisioners:  # Ubuntu images come bundled with GCE integrations provisioned
+provisioners:  # Debian images come bundled with GCE integrations provisioned
     - type: 'shell'
       inline:
         - 'set -e'
@@ -193,17 +193,17 @@ provisioners:  # Ubuntu images come bundled with GCE integrations provisioned
       source: '{{ pwd }}/'
       destination: '/tmp/automation_images/'
 
-    - except: ['ubuntu']
+    - except: ['debian']
       type: 'shell'
       inline:
         - 'set -e'
         - '/bin/bash /tmp/automation_images/base_images/fedora_base-setup.sh'
 
-    - only: ['ubuntu']
+    - only: ['debian']
       type: 'shell'
       inline:
         - 'set -e'
-        - '/bin/bash /tmp/automation_images/base_images/ubuntu_base-setup.sh'
+        - '/bin/bash /tmp/automation_images/base_images/debian_base-setup.sh'
 
 post-processors:
     # Must be double-nested to guarantee execution order

diff --git a/base_images/ubuntu_base-setup.sh → base_images/debian_base-setup.sh b/base_images/ubuntu_base-setup.sh → base_images/debian_base-setup.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# This script is intended to be run by packer, inside an Ubuntu VM.
+# This script is intended to be run by packer, inside an Debian VM.
 # It's purpose is to configure the VM for importing into google cloud,
 # so that it will boot in GCE and be accessable for further use.
 
@@ -16,6 +16,12 @@ REPO_DIRPATH=$(realpath "$SCRIPT_DIRPATH/../")
 # shellcheck source=./lib.sh
 source "$REPO_DIRPATH/lib.sh"
 
+# Switch to Debian Unstable (SID)
+cat << EOF | sudo tee /etc/apt/sources.list
+deb http://deb.debian.org/debian/ unstable main
+deb-src http://deb.debian.org/debian/ unstable main
+EOF
+
 declare -a PKGS
 PKGS=( \
     coreutils
@@ -28,16 +34,11 @@ PKGS=( \
     software-properties-common
 )
 
+echo "Updating package source lists"
 $SUDO apt-get -qq -y update
-
-# At the time of this commit, upgrading past the stock
-# cloud-init (21.3-1-g6803368d-0ubuntu1~21.04.3) causes
-# failure of login w/ new ssh key after reset + reboot.
-if ! ((CONTAINER)); then
-    $SUDO apt-mark hold cloud-init
-fi
-
-$SUDO apt-get -qq -y upgrade
+echo "Upgrading to SID"
+$SUDO apt-get -qq -y full-upgrade
+echo "Installing basic, necessary packages."
 $SUDO apt-get -qq -y install "${PKGS[@]}"
 
 # compatibility / usefullness of all automated scripting (which is bash-centric)

diff --git a/cache_images/cloud.yml b/cache_images/cloud.yml
@@ -19,11 +19,11 @@ variables:  # Empty value means it must be passed in on command-line
     # See Makefile for definitions
     FEDORA_RELEASE: "{{env `FEDORA_RELEASE`}}"
     PRIOR_FEDORA_RELEASE: "{{env `PRIOR_FEDORA_RELEASE`}}"
-    UBUNTU_RELEASE: "{{env `UBUNTU_RELEASE`}}"
+    DEBIAN_RELEASE: "{{env `DEBIAN_RELEASE`}}"
 
 builders:
     - &gce_hosted_image
-      name: 'ubuntu'
+      name: 'debian'
       type: 'googlecompute'
       # N/B: This implies base images always built with same IMG_SFX.
       source_image: '{{ build_name }}-b{{user `IMG_SFX`}}'
@@ -39,7 +39,7 @@ builders:
       labels: &gce_labels # For the VM
         sfx: '{{user `IMG_SFX`}}'
         src: '{{ build_name }}-b{{user `IMG_SFX` }}'
-        release: 'ubuntu-{{user `UBUNTU_RELEASE` | clean_resource_name}}'
+        release: 'debian-{{user `DEBIAN_RELEASE` | clean_resource_name}}'
         stage: cache
       ssh_username: packer  # arbitrary, packer will create & setup w/ temp. keypair
       ssh_pty: 'true'
@@ -179,17 +179,17 @@ provisioners:
       source: '{{ pwd }}/'
       destination: "/tmp/automation_images"
 
-    - except: ['ubuntu']
+    - except: ['debian']
       type: 'shell'
       inline:
         - 'set -e'
         - '/bin/bash /tmp/automation_images/cache_images/fedora_setup.sh'
 
-    - only: ['ubuntu']
+    - only: ['debian']
       type: 'shell'
       inline:
         - 'set -e'
-        - '/bin/bash /tmp/automation_images/cache_images/ubuntu_setup.sh'
+        - '/bin/bash /tmp/automation_images/cache_images/debian_setup.sh'
 
 post-processors:
     # This is critical for human-interaction.  Copntents will be used

diff --git a/cache_images/ubuntu_packaging.sh → cache_images/debian_packaging.sh b/cache_images/ubuntu_packaging.sh → cache_images/debian_packaging.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# This script is called from ubuntu_setup.sh and various Dockerfiles.
+# This script is called from debian_setup.sh and various Dockerfiles.
 # It's not intended to be used outside of those contexts.  It assumes the lib.sh
 # library has already been sourced, and that all "ground-up" package-related activity
 # needs to be done, including repository setup and initial update.
@@ -26,65 +26,14 @@ if [[ "$OS_RELEASE_VER" -le 2004 ]]; then
     lilto ooe.sh $SUDO add-apt-repository --yes ppa:criu/ppa
 fi
 
-# The OpenSuse Open Build System must be utilized to obtain newer
-# development versions of some tools. This helper sets up config
-# files for apt to fetch packages from OBS. We can be called with
-# a variable number of arguments; I think the term is "subprojects"?
-function setup_obs() {
-    # Version of ubuntu, e.g., 22.04
-    local xubuntu_version
-    xubuntu_version="xUbuntu_$(source /etc/os-release; echo $VERSION_ID)"
-
-    local base_url="https://download.opensuse.org/repositories/devel"
-
-    # Assemble the .deb repo URL by appending colon-slash-item for each arg
-    local repo_url="$base_url"
-    local repo_file="/etc/apt/sources.list.d/devel"
-    for i in "$@"; do
-        repo_url+=":/$i"
-        repo_file+=":$i"
-    done
-    repo_url+="/${xubuntu_version}/"
-    repo_file+=":ci.list"
-    echo "deb $repo_url /" | ooe.sh $SUDO tee "$repo_file"
-
-    # GPG key URL is similar to .deb repo, but just colons, no slashes
-    local gpg_url="$base_url"
-    local gpg_file="/etc/apt/trusted.gpg.d/devel"
-    for i in "$@"; do
-        gpg_url+=":$i"
-        gpg_file+="_$i"
-    done
-    gpg_url+="/${xubuntu_version}/Release.key"
-    gpg_file+="_ci.gpg"
-    curl --fail --silent --location --url "$gpg_url" | \
-        gpg --dearmor | \
-        $SUDO tee "$gpg_file" &> /dev/null
-}
-
-# OBS: podman/buildah/skopeo & dependencies, in order to support
-# upstream (i.e. bleeding-edge) development and automated testing.
-# These packages are not otherwise intended for end-user consumption.
-# We expect to need this repo for the foreseeable future.
-# See https://build.opensuse.org/project/show/devel:kubic:libcontainers:unstable
-setup_obs kubic libcontainers unstable
-
-# OBS: FIXME! TEMPORARY! 2022-07-20! Needed because a glibc update broke criu.
-# >>> PLEASE REMOVE THIS ONCE CRIU GETS FIXED IN REGULAR UBUNTU!
-# >>> (No, I -- Ed -- have no idea how to even check that, sorry).
-# Context: https://github.com/containers/podman/pull/14972
-# Context: https://github.com/checkpoint-restore/criu/issues/1935
-setup_obs tools criu
-
-# N/B: DO NOT install the bats package on Ubuntu VMs, it's broken.
-# ref: (still open) https://bugs.launchpad.net/ubuntu/+source/bats/+bug/1882542
 INSTALL_PACKAGES=(\
     apache2-utils
     apparmor
     apt-transport-https
     autoconf
     automake
     bash-completion
+    bats
     bison
     btrfs-progs
     build-essential
@@ -102,11 +51,12 @@ INSTALL_PACKAGES=(\
     emacs-nox
     file
     fuse3
-    git-daemon-run
     gcc
     gettext
+    git-daemon-run
     gnupg2
     go-md2man
+    golang
     iproute2
     iptables
     jq
@@ -176,11 +126,6 @@ INSTALL_PACKAGES=(\
 # Necessary to update cache of newly added repos
 lilto $SUDO apt-get -q -y update
 
-if (($OS_RELEASE_VER==2104)); then
-    echo "Blocking golang-* package interfearance with kubik containers-common"
-    $SUDO apt-mark hold golang-github-containers-common golang-github-containers-image
-fi
-
 echo "Installing general build/testing dependencies"
 bigto $SUDO apt-get -q -y install "${INSTALL_PACKAGES[@]}"
 
@@ -198,10 +143,10 @@ DOWNLOAD_PACKAGES=(\
 )
 
 curl --fail --silent --location \
-    --url  https://download.docker.com/linux/ubuntu/gpg | \
+    --url  https://download.docker.com/linux/debian/gpg | \
     gpg --dearmor | \
     $SUDO tee /etc/apt/trusted.gpg.d/docker_com.gpg &> /dev/null
-echo "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | \
+echo "deb https://download.docker.com/linux/debian $(lsb_release -cs) stable" | \
     ooe.sh $SUDO tee /etc/apt/sources.list.d/docker.list &> /dev/null
 
 if ((CONTAINER==0)) && [[ ${#DOWNLOAD_PACKAGES[@]} -gt 0 ]]; then
@@ -212,18 +157,3 @@ if ((CONTAINER==0)) && [[ ${#DOWNLOAD_PACKAGES[@]} -gt 0 ]]; then
     $SUDO ln -s /var/cache/apt/archives "$PACKAGE_DOWNLOAD_DIR"
     bigto $SUDO apt-get -q -y install --download-only "${DOWNLOAD_PACKAGES[@]}"
 fi
-
-echo "Configuring Go environment"
-# There are multiple (otherwise conflicting) versions of golang available
-# on Ubuntu.  Being primarily localized by env. vars and defaults, dropping
-# a symlink is the appropriate way to "install" a specific version system-wide.
-#
-# Add upstream golang for perf issues
-curl -s -L https://golang.org/dl/go1.18.4.linux-amd64.tar.gz | \
-    $SUDO tar xzf - -C /usr/local/
-# Now linking to upstream golang until ubuntu performance issues are resolved
-$SUDO ln -sf /usr/local/go/bin/* /usr/bin/
-/usr/bin/go version  # make sure it can run
-
-chmod +x $SCRIPT_DIRPATH/podman_tooling.sh
-$SUDO bash $SCRIPT_DIRPATH/podman_tooling.sh