Merge branch 'master' into xcode_offline_role

.github/workflows/build.yml

@@ -28,18 +28,18 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
     - name: Set up Docker Buildx to use cache feature
-      uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v3.1.0
+      uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0
 
     - name: Login to Docker Hub
-      uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3
+      uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3
       with:
         username: ${{ secrets.DOCKER_USERNAME }}
         password: ${{ secrets.DOCKER_TOKEN }}
       if: github.ref == 'refs/heads/master'
 
 
     - name: Docker Build CentOS6 Image Test
-      uses: docker/build-push-action@af5a7ed5ba88268d5278f7203fb52cd833f66d6e # v5.2.0
+      uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
       with:
         file: ./ansible/docker/Dockerfile.CentOS6
         build-args: git_sha=${{ github.sha }}
@@ -50,7 +50,7 @@ jobs:
       if: github.ref != 'refs/heads/master'
 
     - name: Docker Build & Push Centos6 Image to Docker Hub On Merge
-      uses: docker/build-push-action@af5a7ed5ba88268d5278f7203fb52cd833f66d6e # v5.2.0
+      uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
       with:
         file: ./ansible/docker/Dockerfile.CentOS6
         build-args: git_sha=${{ github.sha }}
@@ -68,10 +68,10 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
     - name: Set up Docker Buildx to use cache feature
-      uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v3.1.0
+      uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0
 
     - name: Docker Build Alpine3 Image
-      uses: docker/build-push-action@af5a7ed5ba88268d5278f7203fb52cd833f66d6e # v5.2.0
+      uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
       with:
         file: ./ansible/docker/Dockerfile.Alpine3
         build-args: git_sha=${{ github.sha }}

.github/workflows/build_qemu.yml

@@ -39,7 +39,7 @@ jobs:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
       - name: Run on architecture
-        uses: uraimo/run-on-arch-action@517085f0367c8256bcfa753e3e13e1550af09954 # v2.7.1
+        uses: uraimo/run-on-arch-action@b0ffb25eb00af00468375982384441f063da1741 # v2.7.2
         with:
           arch: ${{ matrix.arch }}
           distro: ${{ matrix.distro }}

.github/workflows/build_vagrant.yml

@@ -24,6 +24,9 @@ jobs:
 
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
+    - name: Update Repos
+      run: sudo apt-get update
+
     - name: Install Ansible
       run: sudo apt-get install ansible
 
@@ -35,7 +38,7 @@ jobs:
 
     - name: Cache Solaris10.box
       id: solaris-10-cache
-      uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4.0.1
+      uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
       with:
         path: Solaris10.box
         key: sol10boxcache

.github/workflows/linter.yml

@@ -42,7 +42,7 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
     - name: Set up Python 3.x
-      uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
+      uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
       with:
         python-version: '3.x'
 

FAQ.md

@@ -85,16 +85,30 @@ have at the moment:
 | Dockerfile | Image | Platforms  | Where is this built? | In use?
 |---|---|---|---|---|
 | [Centos7](./ansible/docker/Dockerfile.CentOS7) | [`adoptopenjdk/centos7_build_image`](https://hub.docker.com/r/adoptopenjdk/centos7_build_image) | linux on amd64, arm64, ppc64le | [Jenkins](https://ci.adoptium.net/job/centos7_docker_image_updater/) | Yes
+| [RHEL7](./ansible/docker/Dockerfile.RHEL7) | n/a - restricted (*) | s390x | [Jenkins](https://ci.adoptium.net/job/rhel7_docker_image_updater/) | Yes
 | [Centos6](./ansible/docker/Dockerfile.CentOS6) | [`adoptopenjdk/centos6_build_image`](https://hub.docker.com/r/adoptopenjdk/centos6_build_image)| linux/amd64 | [GH Actions](.github/workflows/build.yml) | Yes
 | [Alpine3](./ansible/docker/Dockerfile.Alpine3) | [`adoptopenjdk/alpine3_build_image`](https://hub.docker.com/r/adoptopenjdk/alpine3_build_image) | linux/x64 & linux/arm64 | [Jenkins](https://ci.adoptium.net/job/centos7_docker_image_updater/) | Yes
 | [Ubuntu 20.04 (riscv64 only)](./ansible/docker/Dockerfile.Ubuntu2004-riscv64) | [`adoptopenjdk/ubuntu2004_build_image:linux-riscv64`](https://hub.docker.com/r/adoptopenjdk/ubuntu2004_build_image) | linux/riscv64 | [Jenkins](https://ci.adoptium.net/job/centos7_docker_image_updater/) | Yes
 
+<details>
+<summary>(*) - Caveats:</summary>
+
+The RHEL7 image creation for s390x has to be run on a RHEL host using a
+container implementation supplied by Red Hat, and we are using RHEL8 for
+this as it has a stable implemention.  The image creation requires the
+following:
+
+1. The host needs to have an active RHEL subscription
+2. The RHEL7 devkit (which cannot be made public) to be available in a tar file under /usr/local on the host as per the name in the Dockerfile
+</details>
+
 When a change lands into master, the relevant dockerfiles are built using
 the appropriate CI system listed in the table above by configuring them with
-the ansible playbooks and pushing them up to Docker Hub where they can be
-consumed by our jenkins build agents when the `DOCKER_IMAGE` value is
-defined on the jenkins build pipelines as configured in the
-[pipeline_config files](https://github.com/AdoptOpenJDK/ci-jenkins-pipelines/tree/master/pipelines/jobs/configurations).
+the ansible playbooks and - with the exception of the RHEL7 image for s390x -
+pushing them up to Docker Hub where they can be consumed by our jenkins
+build agents when the `DOCKER_IMAGE` value is defined on the jenkins build
+pipelines as configured in the [pipeline_config
+files](https://github.com/AdoptOpenJDK/ci-jenkins-pipelines/tree/master/pipelines/jobs/configurations).
 
 ### Adding a new dockerBuild dockerhub repository
 

ansible/docker/Dockerfile.RHEL7

@@ -1,10 +1,5 @@
 FROM registry.access.redhat.com/rhel7
-# This dockerfile should be built using this from the top level of the repository:
-#  ROSIPW=******* docker build --no-cache -t rhel7_build_image -f ansible/docker/Dockerfile.RHEL7 --build-arg ROSIUSER=******* --secret id=ROSIPW --build-arg git_sha="$(git rev-parse --short HEAD)" `pwd`
-ARG ROSIUSER
-RUN sed -i 's/\(def in_container():\)/\1\n    return False/g' /usr/lib64/python*/*-packages/rhsm/config.py
-RUN --mount=type=secret,id=ROSIPW,required=true subscription-manager register --username=${ROSIUSER} --password="$(cat /run/secrets/ROSIPW)" --auto-attach
-RUN subscription-manager repos --enable rhel-7-for-system-z-optional-rpms
+RUN yum-config-manager --enable rhel-7-for-system-z-optional-rpms
 # ^^ Optional repo needed for Xvfb
 
 ARG git_sha
@@ -15,6 +10,7 @@ RUN yum --enablerepo=rhel-7-server-ansible-2-for-system-z-rpms install -y ansibl
 RUN yum clean all
 
 COPY . /ansible
+COPY devkit /usr/local/devkit
 
 RUN echo "localhost ansible_connection=local" > /ansible/hosts
 
@@ -31,7 +27,10 @@ RUN useradd -c "Jenkins user" -d /home/${user} -u 1002 -g 1003 -m ${user}
 ENV \
     JDK7_BOOT_DIR="/usr/lib/jvm/java-1.7.0-openjdk" \
     JDK8_BOOT_DIR="/usr/lib/jvm/java-1.8.0-openjdk" \
-    JDK10_BOOT_DIR="/usr/lib/jvm/jdk-10" \
     JAVA_HOME="/usr/lib/jvm/java-1.8.0-openjdk"
-RUN subscription-manager unregister
 
+# While this does bloat the image it is required for building the
+# devkit, and the process for that runs as non-root ...
+# Disabled for now as we're going to copy from /usr/local/devkit on the host
+RUN yum clean all
+RUN yum reinstall --downloadonly glibc glibc-headers glibc-devel cups-libs cups-devel libX11 libX11-devel xorg-x11-proto-devel alsa-lib alsa-lib-devel libXext libXext-devel libXtst libXtst-devel libXrender libXrender-devel libXrandr libXrandr-devel freetype freetype-devel libXt libXt-devel libSM libSM-devel libICE libICE-devel libXi libXi-devel libXdmcp libXdmcp-devel libXau libXau-devel libgcc libxcrypt zlib zlib-devel libffi libffi-devel fontconfig fontconfig-devel kernel-headers

ansible/inventory.yml

@@ -70,13 +70,8 @@ hosts:
           solaris10u11-sparcv9-1: {ip: cloud.siteox.com, port: 24322}
 
       - ibmcloud:
-          win2012r2-x64-1: {ip: 169.48.4.138, user: Administrator}
-          win2012r2-x64-2: {ip: 169.48.4.142, user: Administrator}
           win2022-x64-1: {ip: 52.118.206.11, user: Administrator}
 
-      - spearhead:
-          freebsd12-x64-1: {ip: 185.131.222.224}
-
   - docker:
 
       - skytap:
@@ -89,8 +84,6 @@ hosts:
           ubuntu2204-x64-2: {ip: 20.83.24.86, description: 16 cores, 64GB}
 
       - equinix:
-          ubuntu2204-x64-1: {ip: 145.40.113.173, description: Intel Xeon Gold 40 core}
-          ubuntu2004-x64-1: {ip: 145.40.114.58, description: AMD EPYC 7401P 24 core}
           ubuntu2004-armv8-1: {ip: 147.75.35.203, description: Ampere Altra 160 core, 512Gb}
           ubuntu2204-armv8-1: {ip: 139.178.86.243, description: Ampere Altra 160 cores, 512Gb}
 
@@ -113,7 +106,7 @@ hosts:
 
       - azure:
           win2016-x64-1: {ip: 52.149.211.210, user: adoptopenjdk}
-          win2019-x64-1: {ip: 20.185.182.137, user: adoptopenjdk}
+          win2019-x64-1: {ip: 13.92.177.186, user: adoptopenjdk}
           win2022-x64-1: {ip: 51.132.234.42, user: adoptopenjdk}
           win2022-x64-2: {ip: 20.26.116.218, user: adoptopenjdk}
           win11-aarch64-1: {ip: 20.4.31.184, user: adoptopenjdk}
@@ -178,5 +171,3 @@ hosts:
           rhel6-x64-1: {ip: 169.48.4.140}
           rhel7-x64-1: {ip: 169.48.4.136}
           ubuntu1604-x64-1: {ip: 169.48.4.141}
-          win2012r2-x64-1: {ip: 169.48.4.131, user: Administrator}
-          win2012r2-x64-2: {ip: 169.48.4.139, user: Administrator}

ansible/playbooks/AdoptOpenJDK_Unix_Playbook/dockerhost.yml

@@ -33,10 +33,6 @@
       jdk_version: 21
     - role: Nagios_Plugins        # AdoptOpenJDK Infrastructure
       tags: [nagios_plugins, adoptopenjdk]
-    - role: Nagios_Master_Config  # AdoptOpenJDK Infrastructure
-      tags: [nagios_master_config, adoptopenjdk]
-    - role: Nagios_Tunnel         # AdoptOpenJDK Infrastructure
-      tags: [nagios_tunnel, adoptopenjdk]
     - Security
     - Vendor
     - IPv6

ansible/playbooks/AdoptOpenJDK_Unix_Playbook/main.yml

@@ -94,6 +94,7 @@
       when:
         - (ansible_distribution != "Alpine" or ansible_architecture != "aarch64")
         - ansible_architecture != "riscv64"
+        - ansible_architecture != "s390x"
       tags: build_tools
     - role: adoptopenjdk_install  # JDK11 Build Bootstrap
       jdk_version: 10
@@ -120,6 +121,7 @@
         - ansible_distribution != "Solaris"
         - ansible_architecture != "riscv64"
         - ansible_architecture != "armv7l"
+        - ansible_architecture != "s390x"
       tags: build_tools
     - role: adoptopenjdk_install  # Current LTS
       jdk_version: 21
@@ -129,10 +131,6 @@
       tags: build_tools
     - role: Nagios_Plugins        # AdoptOpenJDK Infrastructure
       tags: [nagios_plugins, adoptopenjdk]
-    - role: Nagios_Master_Config  # AdoptOpenJDK Infrastructure
-      tags: [nagios_master_config, adoptopenjdk]
-    - role: Nagios_Tunnel         # AdoptOpenJDK Infrastructure
-      tags: [nagios_tunnel, adoptopenjdk]
     - riscv_cross_compiler        # For building JDK11/J9 on RISC-V architecture
     - Clean_Up
     - Security

ansible/playbooks/AdoptOpenJDK_Unix_Playbook/roles/Docker/tasks/main.yml

@@ -55,6 +55,14 @@
         use: auto       # automatic select package manager to use yum, apt and so on
       when: (ansible_distribution == "Ubuntu" and ansible_distribution_major_version | int >= 20)
 
+    - name: Install docker build on Ubuntu 20+
+      package:
+        name:
+          - docker-buildx
+        state: present
+        use: auto       # automatic select package manager to use yum, apt and so on
+      when: (ansible_distribution == "Ubuntu" and ansible_distribution_major_version | int >= 20)
+
     - name: Install for SLES # zypper does not support in package module
       include_tasks: sles.yml
       when:

ansible/playbooks/AdoptOpenJDK_Unix_Playbook/roles/Xcode_offline/tasks/xcode_offline.yml

@@ -16,7 +16,7 @@
 
     - name: Copy xcode file to remote host
       copy:
-        src: Xcode_{{ xcode_version }}.xip
+        src: /Vendor_Files/macOS/files/Xcode_{{ xcode_version }}.xip
         dest: '~'
         force: false
 

ansible/playbooks/AdoptOpenJDK_Unix_Playbook/roles/deploy_container/tasks/deploy.yml

@@ -22,14 +22,20 @@
 - name: Build {{ docker_image }} docker images
   command: docker build --cpu-period=100000 --cpu-quota=800000 -t aqa_{{ docker_image }} --memory=6G -f /tmp/Dockerfile.{{ docker_image }} /tmp/
 
-# Without specifying a port here, docker will give the container a random unused port
-- name: Run {{ docker_image }} docker container
-  command: docker run --restart unless-stopped -p 22 --cpuset-cpus="0-3" --memory=6G --detach --name {{ docker_image | upper }}.PORT aqa_{{ docker_image }}
+# Finds the highest port number already assigned and +1
+- name: Find available port
+  shell: docker ps --format \"\{\{\.Ports\}\}\" | awk -F[:-] '{print $2}' | sort | tail -n 1
+  register: docker_port_output
+
+- name: Set docker_port variable if empty
+  set_fact:
+    docker_port: 32000
+  when: docker_port_output.stdout == ""
 
-# Now we want to rename the running container with the port name
-- name: Find assigned port of {{ docker_image }} container
-  shell: docker port {{ docker_image | upper }}.PORT | head -n 1 | cut -d ':' -f 2
-  register: docker_port
+- name: Set docker_port variable when non empty
+  set_fact:
+    docker_port: "{{ docker_port_output.stdout | int + 1 }}"
+  when: not (docker_port_output.stdout == "")
 
-- name: Rename {{ docker_image }} container to {{ docker_image | upper }}.{{ docker_port.stdout_lines[0] }}
-  command: docker rename {{ docker_image | upper }}.PORT {{ docker_image | upper }}.{{ docker_port.stdout_lines[0] }}
+- name: Run {{ docker_image }} docker container
+  command: docker run --restart unless-stopped -p {{ docker_port }}:22 --cpuset-cpus="0-3" --memory=6G --detach --name {{ docker_image | upper }}.{{ docker_port }} aqa_{{ docker_image }}