diff --git a/.cirrus.yml b/.cirrus.yml index d6006201..95b0c4d1 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -85,14 +85,16 @@ container_images_task: &container_images env: TARGET_NAME: 'prior-fedora_podman' DEST_FQIN: *fqin - - name: *name - env: - TARGET_NAME: 'ubuntu_podman' - DEST_FQIN: *fqin - name: *name env: TARGET_NAME: 'skopeo_cidev' DEST_FQIN: *fqin + # Upgrading from stable -> SID fails for container-images + # TODO: Re-enable once fixed + # - name: *name + # env: + # TARGET_NAME: 'debian' + # DEST_FQIN: *fqin env: &image_env # For quay.io/libpod namespace REG_USERNAME: ENCRYPTED[de755aef351c501ee480231c24eae25b15e2b2a2b7c629f477c1d427fc5269e360bb358a53bd8914605bae588e99b52a] @@ -163,7 +165,7 @@ base_images_task: PACKER_BUILDS: "fedora-aws-arm64" - <<: *base_image env: - PACKER_BUILDS: "ubuntu" + PACKER_BUILDS: "debian" env: GAC_JSON: &gac_json ENCRYPTED[7fba7fb26ab568ae39f799ab58a476123206576b0135b3d1019117c6d682391370c801e149f29324ff4b50133012aed9] AWS_INI: &aws_ini ENCRYPTED[4cd69097cd29a9899e51acf3bbacceeb83cb5c907d272ca1e2a8ccd515b03f2368a0680870c0d120fc32bc578bb0a930] @@ -219,7 +221,7 @@ cache_images_task: PACKER_BUILDS: "build-push" - <<: *cache_image env: - PACKER_BUILDS: "ubuntu" + PACKER_BUILDS: "debian" env: GAC_JSON: *gac_json AWS_INI: *aws_ini @@ -283,12 +285,12 @@ test_imgts_task: &imgts image-builder-${IMG_SFX} fedora-b${IMG_SFX} prior-fedora-b${IMG_SFX} - ubuntu-b${IMG_SFX} + debian-b${IMG_SFX} fedora-c${IMG_SFX} prior-fedora-c${IMG_SFX} fedora-netavark-c${IMG_SFX} fedora-podman-py-c${IMG_SFX} - ubuntu-c${IMG_SFX} + debian-c${IMG_SFX} build-push-c${IMG_SFX} EC2IMGNAMES: | fedora-aws-b${IMG_SFX} diff --git a/IMG_SFX b/IMG_SFX index 4f0ee425..a93cb437 100644 --- a/IMG_SFX +++ b/IMG_SFX @@ -1 +1 @@ -20230207t150657z-f37f36u2204 +20230215t175929z-f37f36d12 diff --git a/Makefile b/Makefile index f14989ee..d21628f6 100644 --- a/Makefile +++ b/Makefile @@ -23,8 +23,10 @@ export PRIOR_FEDORA_RELEASE = 36 # See import_images/README.md export FEDORA_IMPORT_IMG_SFX = 1669819494 -export UBUNTU_RELEASE = 22.04 -export UBUNTU_BASE_FAMILY = ubuntu-2204-lts +# Automation assumes the actual release number (after SID upgrade) +# is always one-greater than the latest DEBIAN_BASE_FAMILY (GCE image). +export DEBIAN_RELEASE = 12 +export DEBIAN_BASE_FAMILY = debian-11 IMPORT_FORMAT = vhdx @@ -119,7 +121,7 @@ help: ## Default target, parses special in-line comments as documentation. # names and a max-length of 63. .PHONY: IMG_SFX IMG_SFX: ## Generate a new date-based image suffix, store in the file IMG_SFX - $(file >$@,$(shell date --utc +%Y%m%dt%H%M%Sz)-f$(FEDORA_RELEASE)f$(PRIOR_FEDORA_RELEASE)u$(subst .,,$(UBUNTU_RELEASE))) + $(file >$@,$(shell date --utc +%Y%m%dt%H%M%Sz)-f$(FEDORA_RELEASE)f$(PRIOR_FEDORA_RELEASE)d$(subst .,,$(DEBIAN_RELEASE))) @echo "$(file _podman`` 4. [Boot *VMs* from each image produced in step @@ -126,7 +126,7 @@ see step 4 below. * The bulk of the packaging work occurs next, from the `cache_images/*_packaging.sh` scripts. **This is most likely what you want to modify.** -* Unlike the Fedora and Ubuntu scripts, the `build-push` VM image is not +* Unlike the Fedora and Debian scripts, the `build-push` VM image is not for general-purpose use. It's intended to be used by it's embedded `main.sh` script, in downstream repositories for building container images. The image and `main.sh` are both tightly coupled with `build-push` tool @@ -258,13 +258,13 @@ an overview of the process followed **by automation** to produce the ### Podman Several instances of the image-builder VM are used to create container -images. In particular, Fedora and Ubuntu images are created that +images. In particular, Fedora and Debian images are created that more-or-less duplicate the setup of the VM Cache-images. They are then automatically pushed to: * https://quay.io/repository/libpod/fedora_podman * https://quay.io/repository/libpod/prior-fedora_podman -* https://quay.io/repository/libpod/ubuntu_podman +* https://quay.io/repository/libpod/debian_podman The meaning of *prior* and *current*, is defined by the contents of the `*_release` files within the `podman` subdirectory. This is @@ -419,7 +419,7 @@ producing images proceeds as follows: 3. If you wish to build only a subset of available images, list the names you want as comma-separated values of the `PACKER_BUILDS` variable. Be sure you *export* this variable so that `make` has access to it. For - example, `export PACKER_BUILDS=ubuntu,prior-fedora`. + example, `export PACKER_BUILDS=debian,prior-fedora`. 4. Still within the container, again ensure you have plenty of network and CPU resources available. Build the VM Base images by executing the command diff --git a/base_images/cloud-init/debian/cloud.cfg.d/40_defuser.cfg b/base_images/cloud-init/debian/cloud.cfg.d/40_defuser.cfg new file mode 120000 index 00000000..52e43cb5 --- /dev/null +++ b/base_images/cloud-init/debian/cloud.cfg.d/40_defuser.cfg @@ -0,0 +1 @@ +../../fedora/cloud.cfg.d/40_defuser.cfg \ No newline at end of file diff --git a/base_images/cloud-init/debian/cloud.cfg.d/40_enable_root.cfg b/base_images/cloud-init/debian/cloud.cfg.d/40_enable_root.cfg new file mode 120000 index 00000000..98a0e391 --- /dev/null +++ b/base_images/cloud-init/debian/cloud.cfg.d/40_enable_root.cfg @@ -0,0 +1 @@ +../../fedora/cloud.cfg.d/40_enable_root.cfg \ No newline at end of file diff --git a/base_images/cloud-init/fedora/cloud.cfg.d/40_defuser.cfg b/base_images/cloud-init/fedora/cloud.cfg.d/40_defuser.cfg deleted file mode 120000 index 48792ed5..00000000 --- a/base_images/cloud-init/fedora/cloud.cfg.d/40_defuser.cfg +++ /dev/null @@ -1 +0,0 @@ -../../ubuntu/cloud.cfg.d/40_defuser.cfg \ No newline at end of file diff --git a/base_images/cloud-init/fedora/cloud.cfg.d/40_defuser.cfg b/base_images/cloud-init/fedora/cloud.cfg.d/40_defuser.cfg new file mode 100644 index 00000000..12733bb1 --- /dev/null +++ b/base_images/cloud-init/fedora/cloud.cfg.d/40_defuser.cfg @@ -0,0 +1,2 @@ +# Avoid adding any users to the system by default. GCP OSLogin will do it at runtime. +users: {} diff --git a/base_images/cloud-init/fedora/cloud.cfg.d/40_enable_root.cfg b/base_images/cloud-init/fedora/cloud.cfg.d/40_enable_root.cfg deleted file mode 120000 index 7e56fc2f..00000000 --- a/base_images/cloud-init/fedora/cloud.cfg.d/40_enable_root.cfg +++ /dev/null @@ -1 +0,0 @@ -../../ubuntu/cloud.cfg.d/40_enable_root.cfg \ No newline at end of file diff --git a/base_images/cloud-init/fedora/cloud.cfg.d/40_enable_root.cfg b/base_images/cloud-init/fedora/cloud.cfg.d/40_enable_root.cfg new file mode 100644 index 00000000..672d1907 --- /dev/null +++ b/base_images/cloud-init/fedora/cloud.cfg.d/40_enable_root.cfg @@ -0,0 +1 @@ +disable_root: 0 diff --git a/base_images/cloud-init/ubuntu/cloud.cfg.d/40_defuser.cfg b/base_images/cloud-init/ubuntu/cloud.cfg.d/40_defuser.cfg deleted file mode 100644 index 12733bb1..00000000 --- a/base_images/cloud-init/ubuntu/cloud.cfg.d/40_defuser.cfg +++ /dev/null @@ -1,2 +0,0 @@ -# Avoid adding any users to the system by default. GCP OSLogin will do it at runtime. -users: {} diff --git a/base_images/cloud-init/ubuntu/cloud.cfg.d/40_enable_root.cfg b/base_images/cloud-init/ubuntu/cloud.cfg.d/40_enable_root.cfg deleted file mode 100644 index 672d1907..00000000 --- a/base_images/cloud-init/ubuntu/cloud.cfg.d/40_enable_root.cfg +++ /dev/null @@ -1 +0,0 @@ -disable_root: 0 diff --git a/base_images/cloud.yml b/base_images/cloud.yml index 5e5074f8..ecf32b90 100644 --- a/base_images/cloud.yml +++ b/base_images/cloud.yml @@ -28,19 +28,19 @@ variables: # Empty value means it must be passed in on command-line FEDORA_IMPORT_IMG_SFX: "{{env `FEDORA_IMPORT_IMG_SFX`}}" - UBUNTU_RELEASE: "{{env `UBUNTU_RELEASE`}}" - UBUNTU_BASE_FAMILY: "{{env `UBUNTU_BASE_FAMILY`}}" + DEBIAN_RELEASE: "{{env `DEBIAN_RELEASE`}}" + DEBIAN_BASE_FAMILY: "{{env `DEBIAN_BASE_FAMILY`}}" builders: - - name: 'ubuntu' + - name: 'debian' type: 'googlecompute' # Prefix IMG_SFX with "b" so this is never confused with a cache_image name image_name: '{{build_name}}-b{{user `IMG_SFX`}}' image_family: '{{build_name}}-base' image_description: 'Built in https://cirrus-ci.com/task/{{user `CIRRUS_TASK_ID`}}' - source_image_family: '{{user `UBUNTU_BASE_FAMILY`}}' - source_image_project_id: 'ubuntu-os-cloud' + source_image_family: '{{user `DEBIAN_BASE_FAMILY`}}' + source_image_project_id: 'debian-cloud' project_id: '{{user `GCP_PROJECT_ID`}}' # Can't use env. var for this, googlecompute-import only supports filepath account_file: '{{user `GAC_FILEPATH`}}' @@ -49,10 +49,10 @@ builders: # Identify the instance labels: &imgcpylabels sfx: '{{user `IMG_SFX`}}' - src: '{{user `UBUNTU_BASE_FAMILY`}}' + src: '{{user `DEBIAN_BASE_FAMILY`}}' stage: 'base' arch: 'x86_64' - release: 'ubuntu-{{user `UBUNTU_RELEASE` | clean_resource_name}}' + release: 'debian-{{user `DEBIAN_RELEASE` | clean_resource_name}}' # Gotcha: https://www.packer.io/docs/builders/googlecompute#gotchas ssh_username: 'packer' temporary_key_pair_type: ed25519 @@ -183,7 +183,7 @@ builders: snapshot_tags: *awsarm64tags -provisioners: # Ubuntu images come bundled with GCE integrations provisioned +provisioners: # Debian images come bundled with GCE integrations provisioned - type: 'shell' inline: - 'set -e' @@ -193,17 +193,17 @@ provisioners: # Ubuntu images come bundled with GCE integrations provisioned source: '{{ pwd }}/' destination: '/tmp/automation_images/' - - except: ['ubuntu'] + - except: ['debian'] type: 'shell' inline: - 'set -e' - '/bin/bash /tmp/automation_images/base_images/fedora_base-setup.sh' - - only: ['ubuntu'] + - only: ['debian'] type: 'shell' inline: - 'set -e' - - '/bin/bash /tmp/automation_images/base_images/ubuntu_base-setup.sh' + - '/bin/bash /tmp/automation_images/base_images/debian_base-setup.sh' post-processors: # Must be double-nested to guarantee execution order diff --git a/base_images/debian_base-setup.sh b/base_images/debian_base-setup.sh new file mode 100644 index 00000000..e7642ca5 --- /dev/null +++ b/base_images/debian_base-setup.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +# This script is intended to be run by packer, inside an Debian VM. +# It's purpose is to configure the VM for importing into google cloud, +# so that it will boot in GCE and be accessable for further use. + +set -eo pipefail + +SCRIPT_FILEPATH=$(realpath "${BASH_SOURCE[0]}") +SCRIPT_DIRPATH=$(dirname "$SCRIPT_FILEPATH") +REPO_DIRPATH=$(realpath "$SCRIPT_DIRPATH/../") + +# Run as quickly as possible after boot +/bin/bash $REPO_DIRPATH/systemd_banish.sh + +# shellcheck source=./lib.sh +source "$REPO_DIRPATH/lib.sh" + +# Switch to Debian Unstable (SID) +cat << EOF | sudo tee /etc/apt/sources.list +deb http://deb.debian.org/debian/ unstable main +deb-src http://deb.debian.org/debian/ unstable main +EOF + +declare -a PKGS +PKGS=( \ + coreutils + curl + cloud-init + gawk + git + openssh-client + openssh-server + rng-tools5 + software-properties-common +) + +echo "Updating package source lists" +$SUDO apt-get -qq -y update +echo "Upgrading to SID" +$SUDO apt-get -qq -y full-upgrade +echo "Installing basic, necessary packages." +$SUDO apt-get -qq -y install "${PKGS[@]}" + +# compatibility / usefullness of all automated scripting (which is bash-centric) +$SUDO DEBCONF_DB_OVERRIDE='File{'$SCRIPT_DIRPATH/no_dash.dat'}' \ + dpkg-reconfigure dash + +# Ref: https://wiki.debian.org/DebianReleases +# CI automation needs a *sortable* OS version/release number to select/perform/apply +# runtime configuration and workarounds. Since switching to Unstable/SID, a +# numeric release version is not available. While an imperfect solution, +# base an artificial version off the 'base-files' package version, right-padded with +# zeros to ensure sortability (i.e. "12.02" < "12.13"). +base_files_version=$(dpkg -s base-files | awk '/Version:/{print $2}') +base_major=$(cut -d. -f 1 <<<"$base_files_version") +base_minor=$(cut -d. -f 2 <<<"$base_files_version") +sortable_version=$(printf "%02d.%02d" $base_major $base_minor) +echo "WARN: This is NOT an official version number. It's for CI-automation purposes only." +echo "VERSION_ID=\"$sortable_version\"" | \ + $SUDO tee -a /etc/os-release + +install_automation_tooling + +if ! ((CONTAINER)); then + custom_cloud_init + $SUDO systemctl enable rngd + + # Cloud-config fails to enable this for some reason or another + $SUDO sed -i -r \ + -e 's/^PermitRootLogin no/PermitRootLogin prohibit-password/' \ + /etc/ssh/sshd_config +fi + +finalize diff --git a/base_images/ubuntu_base-setup.sh b/base_images/ubuntu_base-setup.sh deleted file mode 100644 index 00b223c4..00000000 --- a/base_images/ubuntu_base-setup.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash - -# This script is intended to be run by packer, inside an Ubuntu VM. -# It's purpose is to configure the VM for importing into google cloud, -# so that it will boot in GCE and be accessable for further use. - -set -eo pipefail - -SCRIPT_FILEPATH=$(realpath "${BASH_SOURCE[0]}") -SCRIPT_DIRPATH=$(dirname "$SCRIPT_FILEPATH") -REPO_DIRPATH=$(realpath "$SCRIPT_DIRPATH/../") - -# Run as quickly as possible after boot -/bin/bash $REPO_DIRPATH/systemd_banish.sh - -# shellcheck source=./lib.sh -source "$REPO_DIRPATH/lib.sh" - -declare -a PKGS -PKGS=( \ - coreutils - curl - gawk - git - openssh-client - openssh-server - rng-tools5 - software-properties-common -) - -$SUDO apt-get -qq -y update - -# At the time of this commit, upgrading past the stock -# cloud-init (21.3-1-g6803368d-0ubuntu1~21.04.3) causes -# failure of login w/ new ssh key after reset + reboot. -if ! ((CONTAINER)); then - $SUDO apt-mark hold cloud-init -fi - -$SUDO apt-get -qq -y upgrade -$SUDO apt-get -qq -y install "${PKGS[@]}" - -# compatibility / usefullness of all automated scripting (which is bash-centric) -$SUDO DEBCONF_DB_OVERRIDE='File{'$SCRIPT_DIRPATH/no_dash.dat'}' \ - dpkg-reconfigure dash - -install_automation_tooling - -if ! ((CONTAINER)); then - custom_cloud_init - $SUDO systemctl enable rngd -fi - -finalize diff --git a/cache_images/cloud.yml b/cache_images/cloud.yml index ba872a5b..d4425625 100644 --- a/cache_images/cloud.yml +++ b/cache_images/cloud.yml @@ -19,11 +19,11 @@ variables: # Empty value means it must be passed in on command-line # See Makefile for definitions FEDORA_RELEASE: "{{env `FEDORA_RELEASE`}}" PRIOR_FEDORA_RELEASE: "{{env `PRIOR_FEDORA_RELEASE`}}" - UBUNTU_RELEASE: "{{env `UBUNTU_RELEASE`}}" + DEBIAN_RELEASE: "{{env `DEBIAN_RELEASE`}}" builders: - &gce_hosted_image - name: 'ubuntu' + name: 'debian' type: 'googlecompute' # N/B: This implies base images always built with same IMG_SFX. source_image: '{{ build_name }}-b{{user `IMG_SFX`}}' @@ -39,7 +39,7 @@ builders: labels: &gce_labels # For the VM sfx: '{{user `IMG_SFX`}}' src: '{{ build_name }}-b{{user `IMG_SFX` }}' - release: 'ubuntu-{{user `UBUNTU_RELEASE` | clean_resource_name}}' + release: 'debian-{{user `DEBIAN_RELEASE` | clean_resource_name}}' stage: cache ssh_username: packer # arbitrary, packer will create & setup w/ temp. keypair ssh_pty: 'true' @@ -179,17 +179,17 @@ provisioners: source: '{{ pwd }}/' destination: "/tmp/automation_images" - - except: ['ubuntu'] + - except: ['debian'] type: 'shell' inline: - 'set -e' - '/bin/bash /tmp/automation_images/cache_images/fedora_setup.sh' - - only: ['ubuntu'] + - only: ['debian'] type: 'shell' inline: - 'set -e' - - '/bin/bash /tmp/automation_images/cache_images/ubuntu_setup.sh' + - '/bin/bash /tmp/automation_images/cache_images/debian_setup.sh' post-processors: # This is critical for human-interaction. Copntents will be used diff --git a/cache_images/debian_packaging.sh b/cache_images/debian_packaging.sh new file mode 100644 index 00000000..d51fceeb --- /dev/null +++ b/cache_images/debian_packaging.sh @@ -0,0 +1,159 @@ +#!/bin/bash + +# This script is called from debian_setup.sh and various Dockerfiles. +# It's not intended to be used outside of those contexts. It assumes the lib.sh +# library has already been sourced, and that all "ground-up" package-related activity +# needs to be done, including repository setup and initial update. + +set -e + +SCRIPT_FILEPATH=$(realpath "${BASH_SOURCE[0]}") +SCRIPT_DIRPATH=$(dirname "$SCRIPT_FILEPATH") +REPO_DIRPATH=$(realpath "$SCRIPT_DIRPATH/../") + +# shellcheck source=./lib.sh +source "$REPO_DIRPATH/lib.sh" + +echo "Updating/Installing repos and packages for $OS_REL_VER" + +lilto ooe.sh $SUDO apt-get -qq -y update +bigto ooe.sh $SUDO apt-get -qq -y upgrade + +echo "Configuring additional package repositories" + +INSTALL_PACKAGES=(\ + apache2-utils + apparmor + apt-transport-https + autoconf + automake + bash-completion + bats + bison + btrfs-progs + build-essential + buildah + bzip2 + ca-certificates + catatonit + conmon + containernetworking-plugins + criu + crun + dnsmasq + e2fslibs-dev + emacs-nox + file + fuse3 + gcc + gettext + git-daemon-run + gnupg2 + go-md2man + golang + iproute2 + iptables + jq + libaio-dev + libapparmor-dev + libbtrfs-dev + libcap-dev + libcap2 + libdevmapper-dev + libdevmapper1.02.1 + libfuse-dev + libfuse2 + libfuse3-dev + libglib2.0-dev + libgpgme11-dev + liblzma-dev + libnet1 + libnet1-dev + libnl-3-dev + libprotobuf-c-dev + libprotobuf-dev + libseccomp-dev + libseccomp2 + libselinux-dev + libsystemd-dev + libtool + libudev-dev + lsb-release + lsof + make + netcat-traditional + openssl + parallel + pkg-config + podman + protobuf-c-compiler + protobuf-compiler + python-is-python3 + python3-dateutil + python3-dateutil + python3-docker + python3-pip + python3-protobuf + python3-psutil + python3-pytoml + python3-requests + python3-setuptools + rsync + runc + scons + skopeo + slirp4netns + socat + systemd-container + sudo + time + unzip + vim + wget + xz-utils + zip + zlib1g-dev + zstd +) + +# Necessary to update cache of newly added repos +lilto $SUDO apt-get -q -y update + +echo "Installing general build/testing dependencies" +bigto $SUDO apt-get -q -y install "${INSTALL_PACKAGES[@]}" + +# Buildah conformance testing needs to install packages from docker.io +# at runtime. Setup the repo here, so it only affects downloaded +# (cached) packages and not updates/installs (above). Installing packages +# cached in the image is preferable to reaching out to the repository +# at runtime. It also has the desirable effect of preventing the +# possibility of package changes from one CI run to the next (or from +# one branch to the next). +DOWNLOAD_PACKAGES=(\ + containerd.io + docker-ce + docker-ce-cli +) + +curl --fail --silent --location \ + --url https://download.docker.com/linux/debian/gpg | \ + gpg --dearmor | \ + $SUDO tee /etc/apt/trusted.gpg.d/docker_com.gpg &> /dev/null + +# Buildah CI does conformance testing vs the most recent Docker version. +# However, there is no Docker release for SID, so just use latest stable +# release for Docker, whatever debian release that cooresponds to. +# Ref: https://wiki.debian.org/DebianReleases +docker_debian_release=bullseye + +echo "deb https://download.docker.com/linux/debian $docker_debian_release stable" | \ + ooe.sh $SUDO tee /etc/apt/sources.list.d/docker.list &> /dev/null + +if ((CONTAINER==0)) && [[ ${#DOWNLOAD_PACKAGES[@]} -gt 0 ]]; then + $SUDO apt-get clean # no reason to keep previous downloads around + # Needed to install .deb files + resolve dependencies + lilto $SUDO apt-get -q -y update + echo "Downloading packages for optional installation at runtime." + $SUDO ln -s /var/cache/apt/archives "$PACKAGE_DOWNLOAD_DIR" + bigto $SUDO apt-get -q -y install --download-only "${DOWNLOAD_PACKAGES[@]}" +fi diff --git a/cache_images/ubuntu_setup.sh b/cache_images/debian_setup.sh similarity index 83% rename from cache_images/ubuntu_setup.sh rename to cache_images/debian_setup.sh index de13f919..177d7d00 100644 --- a/cache_images/ubuntu_setup.sh +++ b/cache_images/debian_setup.sh @@ -1,6 +1,6 @@ #!/bin/bash -# This script is called by packer on the subject Ubuntu VM, to setup the podman +# This script is called by packer on the subject Debian VM, to setup the podman # build/test environment. It's not intended to be used outside of this context. set -e @@ -19,10 +19,10 @@ source "$REPO_DIRPATH/lib.sh" req_env_vars PACKER_BUILD_NAME -bash $SCRIPT_DIRPATH/ubuntu_packaging.sh +bash $SCRIPT_DIRPATH/debian_packaging.sh if ! ((CONTAINER)); then - warn "Making Ubuntu kernel to enable cgroup swap accounting" + warn "Making Debian kernel enable cgroup swap accounting" warn "Forcing CgroupsV1" SEDCMD='s/^GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1 systemd.unified_cgroup_hierarchy=0"/' ooe.sh $SUDO sed -re "$SEDCMD" -i /etc/default/grub.d/* diff --git a/cache_images/fedora_packaging.sh b/cache_images/fedora_packaging.sh index 460620ba..6d5e462a 100644 --- a/cache_images/fedora_packaging.sh +++ b/cache_images/fedora_packaging.sh @@ -216,6 +216,3 @@ cd - # It was observed in F33, dnf install doesn't always get you the latest/greatest lilto $SUDO dnf update -y - -chmod +x $SCRIPT_DIRPATH/podman_tooling.sh -$SUDO $SCRIPT_DIRPATH/podman_tooling.sh diff --git a/cache_images/podman_tooling.sh b/cache_images/podman_tooling.sh deleted file mode 100644 index b62d319c..00000000 --- a/cache_images/podman_tooling.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash - -# This script is executed from *_packaging.sh script to install common/shared -# tooling from the containers/podman repository. It expects -# a go 1.16+ environment has already been set up, and should be called -# as root or through sudo. The script should not be used for any other -# purpose or from any other context. - -set -eo pipefail - -SCRIPT_FILEPATH=$(realpath "${BASH_SOURCE[0]}") -SCRIPT_DIRPATH=$(dirname "$SCRIPT_FILEPATH") -REPO_DIRPATH=$(realpath "$SCRIPT_DIRPATH/../") -# shellcheck source=./lib.sh -source "$REPO_DIRPATH/lib.sh" - -if [[ "$OS_RELEASE_ID" == "ubuntu" ]]; then - if [[ -n "$(type -P bats)" ]]; then - die "Bats _MUST_ not be installed on ubuntu until fixed: https://bugs.launchpad.net/ubuntu/+source/bats/+bug/1882542" - fi - bats_version="1.7.0" - dl_url="https://github.com/bats-core/bats-core/archive/v${bats_version}.tar.gz" - echo "Installing bats $bats_version" - curl --fail --location "$dl_url" | tar xz -C /tmp - pushd /tmp/bats-core-$bats_version - $SUDO ./install.sh /usr/local # prints install location - popd - rm -rf /tmp/bats-core-$bats_version -fi - -echo "Configuring Go environment" -export GOPATH="${GOPATH:-/var/tmp/go}" -mkdir -p "$GOPATH" -export GOSRC=${GOPATH}/src/github.com/containers/podman -export GOCACHE="${GOCACHE:-$GOPATH/cache}" -eval $(go env | tee /dev/stderr) -export PATH="$GOPATH/bin:$PATH" - -# shellcheck disable=SC2154 -if [[ "$OS_RELEASE_ID" == "fedora" ]]; then - if [[ $(uname -m) == "x86_64" ]]; then - msg "Installing swagger binary" - download_url=$(\ - curl -s https://api.github.com/repos/go-swagger/go-swagger/releases/latest | \ - jq -r '.assets[] | select(.name | contains("linux_amd64")) | .browser_download_url') - curl --fail -s -o /usr/local/bin/swagger -L'#' "$download_url" - chmod +x /usr/local/bin/swagger - /usr/local/bin/swagger version - fi -fi - -# Make pristine for other runtime usage/expectations also save a bit -# of space in the images. -rm -rf "$GOPATH/src" "$GOCACHE" -chown -R root.root "$GOPATH" diff --git a/cache_images/ubuntu_packaging.sh b/cache_images/ubuntu_packaging.sh deleted file mode 100644 index 253f7785..00000000 --- a/cache_images/ubuntu_packaging.sh +++ /dev/null @@ -1,229 +0,0 @@ -#!/bin/bash - -# This script is called from ubuntu_setup.sh and various Dockerfiles. -# It's not intended to be used outside of those contexts. It assumes the lib.sh -# library has already been sourced, and that all "ground-up" package-related activity -# needs to be done, including repository setup and initial update. - -set -e - -SCRIPT_FILEPATH=$(realpath "${BASH_SOURCE[0]}") -SCRIPT_DIRPATH=$(dirname "$SCRIPT_FILEPATH") -REPO_DIRPATH=$(realpath "$SCRIPT_DIRPATH/../") - -# shellcheck source=./lib.sh -source "$REPO_DIRPATH/lib.sh" - -echo "Updating/Installing repos and packages for $OS_REL_VER" - -lilto ooe.sh $SUDO apt-get -qq -y update -bigto ooe.sh $SUDO apt-get -qq -y upgrade - -echo "Configuring additional package repositories" - -# Useful version of criu is only available from launchpad repo -if [[ "$OS_RELEASE_VER" -le 2004 ]]; then - lilto ooe.sh $SUDO add-apt-repository --yes ppa:criu/ppa -fi - -# The OpenSuse Open Build System must be utilized to obtain newer -# development versions of some tools. This helper sets up config -# files for apt to fetch packages from OBS. We can be called with -# a variable number of arguments; I think the term is "subprojects"? -function setup_obs() { - # Version of ubuntu, e.g., 22.04 - local xubuntu_version - xubuntu_version="xUbuntu_$(source /etc/os-release; echo $VERSION_ID)" - - local base_url="https://download.opensuse.org/repositories/devel" - - # Assemble the .deb repo URL by appending colon-slash-item for each arg - local repo_url="$base_url" - local repo_file="/etc/apt/sources.list.d/devel" - for i in "$@"; do - repo_url+=":/$i" - repo_file+=":$i" - done - repo_url+="/${xubuntu_version}/" - repo_file+=":ci.list" - echo "deb $repo_url /" | ooe.sh $SUDO tee "$repo_file" - - # GPG key URL is similar to .deb repo, but just colons, no slashes - local gpg_url="$base_url" - local gpg_file="/etc/apt/trusted.gpg.d/devel" - for i in "$@"; do - gpg_url+=":$i" - gpg_file+="_$i" - done - gpg_url+="/${xubuntu_version}/Release.key" - gpg_file+="_ci.gpg" - curl --fail --silent --location --url "$gpg_url" | \ - gpg --dearmor | \ - $SUDO tee "$gpg_file" &> /dev/null -} - -# OBS: podman/buildah/skopeo & dependencies, in order to support -# upstream (i.e. bleeding-edge) development and automated testing. -# These packages are not otherwise intended for end-user consumption. -# We expect to need this repo for the foreseeable future. -# See https://build.opensuse.org/project/show/devel:kubic:libcontainers:unstable -setup_obs kubic libcontainers unstable - -# OBS: FIXME! TEMPORARY! 2022-07-20! Needed because a glibc update broke criu. -# >>> PLEASE REMOVE THIS ONCE CRIU GETS FIXED IN REGULAR UBUNTU! -# >>> (No, I -- Ed -- have no idea how to even check that, sorry). -# Context: https://github.com/containers/podman/pull/14972 -# Context: https://github.com/checkpoint-restore/criu/issues/1935 -setup_obs tools criu - -# N/B: DO NOT install the bats package on Ubuntu VMs, it's broken. -# ref: (still open) https://bugs.launchpad.net/ubuntu/+source/bats/+bug/1882542 -INSTALL_PACKAGES=(\ - apache2-utils - apparmor - apt-transport-https - autoconf - automake - bash-completion - bison - btrfs-progs - build-essential - buildah - bzip2 - ca-certificates - catatonit - conmon - containernetworking-plugins - containers-common - criu - crun - dnsmasq - e2fslibs-dev - emacs-nox - file - fuse3 - git-daemon-run - gcc - gettext - gnupg2 - go-md2man - iproute2 - iptables - jq - libaio-dev - libapparmor-dev - libbtrfs-dev - libcap-dev - libcap2 - libdevmapper-dev - libdevmapper1.02.1 - libfuse-dev - libfuse2 - libfuse3-dev - libglib2.0-dev - libgpgme11-dev - liblzma-dev - libnet1 - libnet1-dev - libnl-3-dev - libprotobuf-c-dev - libprotobuf-dev - libseccomp-dev - libseccomp2 - libselinux-dev - libsystemd-dev - libtool - libudev-dev - lsb-release - lsof - make - netcat - openssl - parallel - pkg-config - podman - podman-plugins - protobuf-c-compiler - protobuf-compiler - python-is-python3 - python2 - python3-dateutil - python3-dateutil - python3-docker - python3-pip - python3-protobuf - python3-psutil - python3-pytoml - python3-requests - python3-setuptools - rsync - runc - scons - skopeo - slirp4netns - socat - sudo - time - unzip - vim - wget - xz-utils - zip - zlib1g-dev - zstd -) - -# Necessary to update cache of newly added repos -lilto $SUDO apt-get -q -y update - -if (($OS_RELEASE_VER==2104)); then - echo "Blocking golang-* package interfearance with kubik containers-common" - $SUDO apt-mark hold golang-github-containers-common golang-github-containers-image -fi - -echo "Installing general build/testing dependencies" -bigto $SUDO apt-get -q -y install "${INSTALL_PACKAGES[@]}" - -# Buildah conformance testing needs to install packages from docker.io -# at runtime. Setup the repo here, so it only affects downloaded -# (cached) packages and not updates/installs (above). Installing packages -# cached in the image is preferable to reaching out to the repository -# at runtime. It also has the desirable effect of preventing the -# possibility of package changes from one CI run to the next (or from -# one branch to the next). -DOWNLOAD_PACKAGES=(\ - containerd.io - docker-ce - docker-ce-cli -) - -curl --fail --silent --location \ - --url https://download.docker.com/linux/ubuntu/gpg | \ - gpg --dearmor | \ - $SUDO tee /etc/apt/trusted.gpg.d/docker_com.gpg &> /dev/null -echo "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | \ - ooe.sh $SUDO tee /etc/apt/sources.list.d/docker.list &> /dev/null - -if ((CONTAINER==0)) && [[ ${#DOWNLOAD_PACKAGES[@]} -gt 0 ]]; then - $SUDO apt-get clean # no reason to keep previous downloads around - # Needed to install .deb files + resolve dependencies - lilto $SUDO apt-get -q -y update - echo "Downloading packages for optional installation at runtime." - $SUDO ln -s /var/cache/apt/archives "$PACKAGE_DOWNLOAD_DIR" - bigto $SUDO apt-get -q -y install --download-only "${DOWNLOAD_PACKAGES[@]}" -fi - -echo "Configuring Go environment" -# There are multiple (otherwise conflicting) versions of golang available -# on Ubuntu. Being primarily localized by env. vars and defaults, dropping -# a symlink is the appropriate way to "install" a specific version system-wide. -# -# Add upstream golang for perf issues -curl -s -L https://golang.org/dl/go1.18.4.linux-amd64.tar.gz | \ - $SUDO tar xzf - -C /usr/local/ -# Now linking to upstream golang until ubuntu performance issues are resolved -$SUDO ln -sf /usr/local/go/bin/* /usr/bin/ -/usr/bin/go version # make sure it can run - -chmod +x $SCRIPT_DIRPATH/podman_tooling.sh -$SUDO bash $SCRIPT_DIRPATH/podman_tooling.sh diff --git a/lib.sh b/lib.sh index 45d9b0fc..2f2ba968 100644 --- a/lib.sh +++ b/lib.sh @@ -1,7 +1,7 @@ # This file is intended to be sourced by other scripts running on -# aa Fedora or Ubuntu VM during various stages of initial setup. +# aa Fedora or Debian VM during various stages of initial setup. # Using it in any other way or context unlikely to do anything # useful for you. @@ -13,7 +13,7 @@ OS_RELEASE_ID="$(source /etc/os-release; echo $ID)" OS_REL_VER="$OS_RELEASE_ID-$OS_RELEASE_VER" # Avoid getting stuck waiting for user input -[[ "$OS_RELEASE_ID" != "ubuntu" ]] || \ +[[ "$OS_RELEASE_ID" != "debian" ]] || \ export DEBIAN_FRONTEND="noninteractive" # This location is checked by automation in other repos, please do not change. @@ -35,7 +35,7 @@ if [[ -n "$UID" ]] && [[ "$UID" -ne 0 ]]; then SUDO="sudo" fi -if [[ "$OS_RELEASE_ID" == "ubuntu" ]]; then +if [[ "$OS_RELEASE_ID" == "debian" ]]; then export DEBIAN_FRONTEND=noninteractive SUDO="$SUDO env DEBIAN_FRONTEND=$DEBIAN_FRONTEND" fi @@ -74,6 +74,7 @@ custom_cloud_init() { if [[ -d "$CUSTOM_CLOUD_CONFIG_DEFAULTS" ]] then echo "Installing custom cloud-init defaults" + mkdir -p /etc/cloud/cloud.cfg.d # Should exist, sometimes doesn't. $SUDO cp -v --dereference \ "$CUSTOM_CLOUD_CONFIG_DEFAULTS"/* \ /etc/cloud/cloud.cfg.d/ @@ -221,9 +222,9 @@ remove_netavark_aardvark_files() { req_env_vars OS_RELEASE_ID # OS_RELEASE_ID is defined by automation-library # shellcheck disable=SC2154 - if [[ "$OS_RELEASE_ID" =~ "ubuntu" ]] + if [[ "$OS_RELEASE_ID" =~ "debian" ]] then - die "Ubuntu netavark/aardvark-dns testing is not supported" + die "Debian netavark/aardvark-dns testing is not supported" fi LISTING_CMD="rpm -ql podman" @@ -256,7 +257,7 @@ clean_automatic_users() { DELUSER="userdel --remove"; DELGROUP="groupdel" fi - # Avoid needing to parse login.defs (fedora) and deluser.conf (Ubuntu) + # Avoid needing to parse login.defs (fedora) and deluser.conf (Debian) # for the UID/GID ranges standard user accounts. cd /home || exit for account in *; do @@ -319,11 +320,11 @@ rh_finalize() { } # Called during VM Image setup, not intended for general use. -ubuntu_finalize() { +debian_finalize() { set +e # Don't fail at the very end # N/B: Several CI setups depend on VMs with downloaded/cached # packages under /var/cache/download a.k.a. /var/cache/apt/archives. - # Avoid apt cache cleaning on Ubuntu VMs! + # Avoid apt cache cleaning on Debian VMs! if ((CONTAINER)); then # try to save a little space for containers msg "Cleaning up packaging metadata and cache" $SUDO apt-get clean @@ -339,8 +340,8 @@ finalize() { rh_finalize elif [[ "$OS_RELEASE_ID" == "fedora" ]]; then rh_finalize - elif [[ "$OS_RELEASE_ID" == "ubuntu" ]]; then - ubuntu_finalize + elif [[ "$OS_RELEASE_ID" == "debian" ]]; then + debian_finalize else die "Unknown/Unsupported Distro '$OS_RELEASE_ID'" fi diff --git a/podman/Containerfile b/podman/Containerfile index 1e99b99f..7fb72f05 100644 --- a/podman/Containerfile +++ b/podman/Containerfile @@ -4,7 +4,7 @@ # top-level Makefile. # N/B: The BASE_NAME and BASE_TAG build-vars are required. Supported -# values for Fedora and Ubuntu, are specified in the *_release +# values for Fedora and Debian, are specified in the *_release # files within this dir. ARG BASE_NAME= diff --git a/podman/setup.sh b/podman/setup.sh index a3a6cf48..6c944c00 100644 --- a/podman/setup.sh +++ b/podman/setup.sh @@ -12,9 +12,9 @@ REPO_DIRPATH=$(realpath "$SCRIPT_DIRPATH/../") # shellcheck source=./lib.sh source "$REPO_DIRPATH/lib.sh" -if [[ "$OS_RELEASE_ID" == "ubuntu" ]]; then - bash base_images/ubuntu_base-setup.sh - bash cache_images/ubuntu_setup.sh +if [[ "$OS_RELEASE_ID" == "debian" ]]; then + bash base_images/debian_base-setup.sh + bash cache_images/debian_setup.sh elif [[ "$OS_RELEASE_ID" == "fedora" ]]; then bash base_images/fedora_base-setup.sh bash cache_images/fedora_setup.sh diff --git a/systemd_banish.sh b/systemd_banish.sh index 14678844..d1634d02 100644 --- a/systemd_banish.sh +++ b/systemd_banish.sh @@ -1,6 +1,6 @@ #!/bin/bash -# This is intended to be executed stand-alone, on a Fedora or Ubuntu VM +# This is intended to be executed stand-alone, on a Fedora or Debian VM # by automation. Alternatively, it may be executed with the '--list' # option to return the list of systemd units defined for disablement # (useful for testing). @@ -32,7 +32,7 @@ do ) &> /dev/null done -# Sigh, for Ubuntu the above isn't enough. There are also periodic apt jobs. +# Sigh, for Debian the above isn't enough. There are also periodic apt jobs. EAAD="/etc/apt/apt.conf.d" PERIODIC_APT_RE='^(APT::Periodic::.+")1"\;' if [[ -d "$EAAD" ]]; then