[WIP] Switch to centos based dockerfile.

galaxy_importer/ansible_test/builders/local_image_build.py

@@ -15,6 +15,7 @@
 # You should have received a copy of the Apache License
 # along with Galaxy.  If not, see <http://www.apache.org/licenses/>.
 
+import glob
 import logging
 import os
 import pkg_resources
@@ -91,17 +92,26 @@ def _build_dockerfile(dir):
             f.writelines(lines)
 
     def _build_image_with_artifact(container_engine, dir):
-        pkg_entrypoint = pkg_resources.resource_filename(
-            "galaxy_importer", "ansible_test/container/entrypoint.sh"
-        )
-        eda_linting = pkg_resources.resource_filename(
-            "galaxy_importer", "ansible_test/container/eda/tox.ini"
+        dockerfile = pkg_resources.resource_filename(
+            "galaxy_importer", "ansible_test/container/Dockerfile"
         )
-        shutil.copyfile(pkg_entrypoint, os.path.join(dir, "entrypoint.sh"))
-        os.mkdir(os.path.join(dir, "eda"))
-        shutil.copyfile(eda_linting, os.path.join(dir, "eda", "tox.ini"))
+        src = os.path.dirname(dockerfile)
+        paths = glob.glob(f"{src}/*")
+        for path in paths:
+            # the dockerfile is modified in a previous function, so
+            # we can't just overwrite it with the original.
+            if os.path.basename(path) == "Dockerfile":
+                continue
+            dst = os.path.join(dir, os.path.basename(path))
+            print(f"{path} > {dst}")
+            if os.path.isdir(path):
+                shutil.copytree(path, dst)
+            else:
+                shutil.copyfile(path, dst)
+
+        # cmd = [container_engine, "build", ".", "--quiet"]
+        cmd = [container_engine, "build", "."]
 
-        cmd = [container_engine, "build", ".", "--quiet"]
         proc = Popen(
             cmd,
             cwd=dir,
@@ -111,13 +121,24 @@ def _build_image_with_artifact(container_engine, dir):
         )
 
         image_id = ""
+        build_log = []
         for line in proc.stdout:
-            image_id = line.strip()
+            line = line.rstrip()
+            if not line:
+                continue
+            print(line)
+            build_log.append(line)
+            if "sha256:" in line and "FROM" not in line:
+                words = line.split()
+                words = [x for x in words if x.startswith("sha256:")]
+                image_id = words[0].replace("sha256:", "")
 
         return_code = proc.wait()
         if return_code != 0:
             raise exceptions.AnsibleTestError(
-                "An exception occurred in {}, returncode={}".format(" ".join(cmd), return_code)
+                "An exception occurred in {}, returncode={} {}".format(
+                    " ".join(cmd), return_code, "\n".join(build_log)
+                )
             )
         if container_engine == "docker":
             image_id = image_id.split(":")[-1]

galaxy_importer/ansible_test/container/Dockerfile

@@ -1,34 +1,91 @@
-FROM quay.io/ansible/base-test-container:4.1.0
+FROM quay.io/centos/centos:stream9
 
-COPY entrypoint.sh /entrypoint
+RUN rpm -e --nodeps curl-minimal
+RUN dnf install -y epel-release
+RUN dnf install -y \
+    ca-certificates \
+    curl \
+    g++ \
+    gcc \
+    git \
+    gnupg2 \
+    bzip2-devel \
+    libffi-devel \
+    readline-devel \
+    sqlite-devel \
+    libxml2-devel \
+    libxslt-devel \
+    libyaml \
+    glibc-locale-source \
+    make \
+    openssh-clients \
+    openssh-server \
+    openssl \
+    python3-pip \
+    python3-devel \
+    python3-distlib \
+    python3.11 \
+    python3.11-pip \
+    python3.11-devel \
+    libicu \
+    libicu-devel \
+    shellcheck \
+    sudo \
+    systemd-sysv \
+    wget \
+    openssl \
+    openssl-devel \
+    patch
+
+# need py 3.6, 3.7, 3.10, 3.12
+COPY files/install_python.sh /usr/bin/install_python.sh
+RUN chmod +x /usr/bin/install_python.sh
+RUN /usr/bin/install_python.sh 2.7
+RUN /usr/bin/install_python.sh 3.5
+RUN /usr/bin/install_python.sh 3.6
+RUN /usr/bin/install_python.sh 3.7
+RUN /usr/bin/install_python.sh 3.8
+RUN /usr/bin/install_python.sh 3.10
+
+RUN alternatives --install /usr/bin/python python /usr/bin/python3.11 1
+RUN alternatives --set python /usr/bin/python3.11
+
+# powershell sanity
+COPY files/install_powershell.sh /tmp/install_powershell.sh
+RUN chmod +x /tmp/install_powershell.sh; /tmp/install_powershell.sh
+
+# populate ansible-test paths and files
+RUN git clone https://github.com/ansible/base-test-container /tmp/base-test-container
+RUN ls -al /tmp/base-test-container
+RUN mkdir -p /usr/share/container-setup
+RUN cp /tmp/base-test-container/files/*.py /usr/share/container-setup/
+RUN ln -s /usr/bin/python3.11 /usr/share/container-setup/python
+
+########################################
+# galaxy-importer specific
+########################################
 
-RUN useradd user1 \
-      --uid 1000 \
-      --no-create-home \
-      --gid root && \
-    apt-get update -y && \
-    # upgrade all packages except python, as they can conflict with python setup in base image
-    apt-mark hold python* && \
-    apt-get upgrade -y && \
-    apt-get install -y wget && \
-    chmod +x /entrypoint && \
-    mkdir -m 0775 /archive && \
-    mkdir -p -m 0775 /ansible_collections /ansible_collections/ns /ansible_collections/ns/col && \
-    touch /ansible_collections/ns/col/placeholder.txt && \
-    # On updating ansible-core version, update the FROM statement to the matching base-test-container version
-    python3.9 -m pip install ansible-core==2.15.0 --disable-pip-version-check && \
-    python3.9 -m pip install tox && \
-    # Creates dir with correct permissions for where ansible-test sanity writes files, needed for non-privileged containers
-    mkdir -m 0775 -p /.cache/pylint && \
-    mkdir -m 0775 -p /eda /eda/tox
+RUN mkdir -m 0775 -p /archive
+RUN mkdir -m 0775 -p /.cache/pylint
+RUN mkdir -m 0775 -p /eda/tox
 
+RUN mkdir -m 0775 -p /ansible_collections/ns/col
+RUN touch /ansible_collections/ns/col/placeholder.txt
+
+RUN useradd user1 --uid=1001 --no-create-home --gid root
+RUN chown -R user1 /ansible_collections
+RUN ls -al /
+#RUN exit 1
+
+RUN python3.9 -m pip install ansible-core==2.15.0 --disable-pip-version-check
+RUN python3.9 -m pip install tox
+
+COPY entrypoint.sh /entrypoint
+RUN chmod +x /entrypoint
 COPY eda/tox.ini /eda/tox
 
 ENV HOME /
-
-RUN cd /ansible_collections/ns/col && \
-    ansible-test sanity --prime-venvs && \
-    chmod -R 0775 /.ansible
+RUN cd /ansible_collections/ns/col && HOME=/ ansible-test sanity --prime-venvs && chmod -R 0775 /.ansible
 
 ENTRYPOINT ["/entrypoint"]
 

galaxy_importer/ansible_test/container/entrypoint.sh

@@ -96,4 +96,3 @@ then
 else
     echo "No EDA content found. Skipping linters."
 fi
-

galaxy_importer/ansible_test/container/files/install_powershell.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+# Install PowerShell using a binary archive.
+# This allows pinning to a specific version, and also brings support for multiple architectures.
+version="7.3.3"
+major_version="$(echo ${version} | cut -f 1 -d .)"
+install_dir="/opt/microsoft/powershell/${major_version}"
+tmp_file="/tmp/powershell.tgz"
+arch="$(uname -i)"
+arch=$(if [ "${arch}" = "x86_64" ]; then echo "x64"; \
+     elif [ "${arch}" = "aarch64" ]; then echo "arm64"; \
+     else echo "unknown arch: ${arch}" && exit 1; fi)
+url="https://github.com/PowerShell/PowerShell/releases/download/v${version}/powershell-${version}-linux-${arch}.tar.gz"
+echo "URL: ${url}"
+curl -sL "${url}" > "${tmp_file}"
+mkdir -p "${install_dir}"
+tar zxf "${tmp_file}" --no-same-owner --no-same-permissions -C "${install_dir}"
+rm "${tmp_file}"
+find "${install_dir}" -type f -exec chmod -x "{}" ";"
+chmod +x "${install_dir}/pwsh"
+ln -s "${install_dir}/pwsh" /usr/bin/pwsh
+pwsh --version

galaxy_importer/ansible_test/container/files/install_python.sh

@@ -0,0 +1,16 @@
+#/bin/bash
+
+set -e
+
+if [ ! -d /.pyenv ]; then
+    git clone https://github.com/pyenv/pyenv.git /.pyenv
+    cd /.pyenv && src/configure && make -C src
+fi
+
+rm -rf /tmp/python-build*
+export PYENV_ROOT=/.pyenv
+export PATH="$PYENV_ROOT/bin:$PATH"
+eval "$(pyenv init -)"
+pyenv install -v $1
+alternatives --install /usr/bin/python$1 python$1 /.pyenv/versions/$1.*/bin/python 1
+alternatives --install /usr/bin/pip$1 pip$1 /.pyenv/versions/$1.*/bin/python 1