diff --git a/ansible/MANUAL_STEPS.md b/ansible/MANUAL_STEPS.md index 4118be513..112d39d97 100644 --- a/ansible/MANUAL_STEPS.md +++ b/ansible/MANUAL_STEPS.md @@ -50,6 +50,14 @@ To add an entry do the following: ## `release-*` machines +Release machines must be able to upload release artifacts to the nodejs.org +web server. The [release-builder](roles/release-builder) Ansible role will +write the necessary key and ssh config onto the release machine, automating +the previously manual steps. + +
+Manual steps + Once setup, they must have `~iojs/.ssh` cloned from another machine, so they have the ssh setup and keys required to upload release artifacts to the nodejs.org web server. The result will be 2 files, an `id_rsa` containing @@ -72,6 +80,7 @@ the release builds will fail. After setting up .ssh, do something like this: ssh node-www date // ... accept the host keys ``` +
### `release-*container*` machines diff --git a/ansible/README.md b/ansible/README.md index bfc98133e..e405d586d 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -234,10 +234,10 @@ debugging problems, see the Unsorted stuff of things we need to do/think about -- [ ] playbook: copy keys and config to release machines +- [x] playbook: copy keys and config to release machines - [ ] avoid messing with keys on machines that has multiple usage such as jump hosts (or set up a new jump host) -- [ ] copy release (staging) keys to release machines +- [x] copy release (staging) keys to release machines - [ ] backup host: generate config, install rsnapshot - [ ] switch to slaveLog for all jenkins instances lacking stdout redirection (note: this depends on init type!) @@ -262,7 +262,7 @@ Unsorted stuff of things we need to do/think about setup/raspberry-pi/README.md, some of these can be automated) - [ ] epel-release for centos - required for centos7 on packet.net arm64 before ccache can be installed -- [ ] make .ssh/config and .ssh/id_rsa for release machines, adding config +- [x] make .ssh/config and .ssh/id_rsa for release machines, adding config for `node-www` and record host key for node-www - [ ] add explicit ARCH and DESTCPU for release machines (RV: I'm adding "arm64" manually for both to force the right thing, from memory I've diff --git a/ansible/inventory.yml b/ansible/inventory.yml index 3247e5247..1a8af4d80 100644 --- a/ansible/inventory.yml +++ b/ansible/inventory.yml @@ -46,6 +46,7 @@ hosts: ip: 169.48.19.173 server_jobs: 6 rhel7-s390x-1: {ip: 148.100.86.101, user: linux1} + rhel8-s390x-1: {ip: 148.100.84.27, user: linux1} - iinthecloud: ibmi73-ppc64_be-1: {ip: 65.183.160.62, user: nodejs} @@ -152,6 +153,10 @@ hosts: rhel7-s390x-2: {ip: 148.100.86.117, user: linux1, build_test_v8: yes} rhel7-s390x-3: {ip: 148.100.86.28, user: linux1, build_test_v8: yes} rhel7-s390x-4: {ip: 148.100.86.94, user: linux1, build_test_v8: yes} + rhel8-s390x-1: {ip: 148.100.84.112, user: linux1, build_test_v8: yes} + rhel8-s390x-2: {ip: 148.100.84.240, user: linux1, build_test_v8: yes} + rhel8-s390x-3: {ip: 148.100.84.56, user: linux1, build_test_v8: yes} + rhel8-s390x-4: {ip: 148.100.84.150, user: linux1, build_test_v8: yes} ubuntu1804-x64-1: {ip: 52.117.26.14, alias: jenkins-workspace-6} ubuntu1804-x64-2: {ip: 50.97.245.9} diff --git a/ansible/playbooks/jenkins/worker/create.yml b/ansible/playbooks/jenkins/worker/create.yml index 17c057be1..5310fb96c 100644 --- a/ansible/playbooks/jenkins/worker/create.yml +++ b/ansible/playbooks/jenkins/worker/create.yml @@ -16,6 +16,8 @@ - { role: 'benchmarking', when: is_benchmark is defined and is_benchmark|bool == True } - jenkins-worker + - { role: release-builder, + when: '"release" in group_names' } pre_tasks: # Requires `secret: XXX` to be in the ansible/host_vars/HOST diff --git a/ansible/roles/baselayout/tasks/main.yml b/ansible/roles/baselayout/tasks/main.yml index f7cb77c01..7e74b2d55 100644 --- a/ansible/roles/baselayout/tasks/main.yml +++ b/ansible/roles/baselayout/tasks/main.yml @@ -141,6 +141,16 @@ state: link src: "/usr/local/bin/python2" +# Required for V8 builds +- name: rhel8 | update python package alternatives + community.general.alternatives: + link: /usr/bin/python + name: python + path: /usr/bin/python2 + when: + - os == "rhel8" + - build_test_v8|default(False) + - name: smartos17 | update gcc symlinks when: os == "smartos17" file: diff --git a/ansible/roles/baselayout/tasks/partials/repo/rhel8.yml b/ansible/roles/baselayout/tasks/partials/repo/rhel8.yml new file mode 100644 index 000000000..094696678 --- /dev/null +++ b/ansible/roles/baselayout/tasks/partials/repo/rhel8.yml @@ -0,0 +1,14 @@ +--- + +# Red Hat Enterprise Linux 8 + +- name: install GPG key for EPEL 8 + become: yes + ansible.builtin.rpm_key: + key: https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-8 + state: present + +- name: install EPEL 8 + ansible.builtin.dnf: + name: https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm + state: present diff --git a/ansible/roles/baselayout/vars/main.yml b/ansible/roles/baselayout/vars/main.yml index fb7ef11e2..9ba2cfd12 100644 --- a/ansible/roles/baselayout/vars/main.yml +++ b/ansible/roles/baselayout/vars/main.yml @@ -144,6 +144,14 @@ packages: { 'gcc-c++,sudo,git,zip,unzip,iptables-services,GConf2-devel,openssl-devel,python3', ], + rhel8_s390x: [ + 'GConf2-devel,python2' # Needed for V8 builds + ], + + rhel8: [ + 'ccache,cmake,gcc-c++,gcc-toolset-11,git,make,python3', + ], + smartos: [ 'gccmakedep', 'git', diff --git a/ansible/roles/bootstrap/handlers/main.yml b/ansible/roles/bootstrap/handlers/main.yml new file mode 100644 index 000000000..1aa2ad684 --- /dev/null +++ b/ansible/roles/bootstrap/handlers/main.yml @@ -0,0 +1,4 @@ +- name: restart iptables + ansible.builtin.service: + name: iptables + state: restarted diff --git a/ansible/roles/bootstrap/tasks/main.yml b/ansible/roles/bootstrap/tasks/main.yml index c10e878a5..f457a43f4 100644 --- a/ansible/roles/bootstrap/tasks/main.yml +++ b/ansible/roles/bootstrap/tasks/main.yml @@ -10,6 +10,7 @@ loop_var: bootstrap_include with_first_found: - files: + - "{{ role_path }}/tasks/partials/{{ os }}-{{ arch }}.yml" - "{{ role_path }}/tasks/partials/{{ os }}.yml" - "{{ role_path }}/tasks/partials/{{ os|stripversion }}.yml" skip: true diff --git a/ansible/roles/bootstrap/tasks/partials/rhel8-s390x.yml b/ansible/roles/bootstrap/tasks/partials/rhel8-s390x.yml new file mode 100644 index 000000000..42fb155f6 --- /dev/null +++ b/ansible/roles/bootstrap/tasks/partials/rhel8-s390x.yml @@ -0,0 +1,42 @@ +--- + +# Red Hat Enterprise Linux 8 on s390x (LinuxONE) + +- name: run common RHEL 8 tasks + ansible.builtin.include_tasks: rhel8.yml + +- name: Firewall | install iptables-services + ansible.builtin.dnf: + name: iptables-services + state: present + +- name: Firewall | enable iptables + ansible.builtin.systemd: + enabled: yes + name: iptables + +- name: Firewall | remove firewalld + ansible.builtin.dnf: + name: firewalld + state: absent + +- name: Firewall | add rule to allow accepting multicast + lineinfile: + dest: /etc/sysconfig/iptables + insertafter: ":OUTPUT ACCEPT.*]" + line: "-A INPUT -m pkttype --pkt-type multicast -j ACCEPT" + notify: restart iptables + +- name: Firewall | add basic rule to allow communication locally + lineinfile: + dest: /etc/sysconfig/iptables + insertafter: ":OUTPUT ACCEPT.*]" + line: "-A INPUT -s 127.0.0.1/32 -d 127.0.0.1/32 -j ACCEPT" + notify: restart iptables + +- name: Firewall | add additional rule to allow communication from 127.0.0.2 + lineinfile: + dest: /etc/sysconfig/iptables + insertafter: ":OUTPUT ACCEPT.*]" + line: "-A INPUT -s 127.0.0.2/32 -d 127.0.0.1/32 -j ACCEPT" + notify: restart iptables diff --git a/ansible/roles/bootstrap/tasks/partials/rhel8.yml b/ansible/roles/bootstrap/tasks/partials/rhel8.yml new file mode 100644 index 000000000..2be829594 --- /dev/null +++ b/ansible/roles/bootstrap/tasks/partials/rhel8.yml @@ -0,0 +1,9 @@ +--- + +# Red Hat Enterprise Linux 8 + +- name: register Red Hat subscription + community.general.redhat_subscription: + activationkey: "{{ type }}" + org_id: "{{ rh_org }}" + state: present diff --git a/ansible/roles/gn/tasks/main.yml b/ansible/roles/gn/tasks/main.yml index d1f8337e8..875aaa964 100644 --- a/ansible/roles/gn/tasks/main.yml +++ b/ansible/roles/gn/tasks/main.yml @@ -8,6 +8,7 @@ ansible.builtin.git: dest: "{{ gn_git_dir }}" repo: "https://gn.googlesource.com/gn" + version: "{{ gn_version|default(omit) }}" become: "{{ gn_user|default(omit)|bool }}" become_user: "{{ gn_user|default(omit) }}" register: gn_git @@ -23,7 +24,7 @@ - name: build gn ansible.builtin.shell: | python3 build/gen.py && \ - . /opt/rh/devtoolset-8/enable && \ + {{ gn_select_compiler }} && \ {{ gn_dest_dir }}/ninja -C out && \ out/gn_unittests args: diff --git a/ansible/roles/gn/vars/main.yml b/ansible/roles/gn/vars/main.yml new file mode 100644 index 000000000..909a5bd19 --- /dev/null +++ b/ansible/roles/gn/vars/main.yml @@ -0,0 +1,12 @@ +--- + +compiler: { + 'centos7': '. /opt/rh/devtoolset-8/enable', + 'rhel7': '. /opt/rh/devtoolset-8/enable', + 'rhel8': '. /opt/rh/gcc-toolset-11/enable' +} + +gn_select_compiler: "{{ compiler[os]|default(compiler[os|stripversion])|default('true') }}" +# Pin gn for now so we can still build older versions of V8 in Node.js 14. +# Refs: https://github.com/nodejs/node/pull/40689#issuecomment-956303875 +gn_version: 69ec4fc diff --git a/ansible/roles/java-base/tasks/main.yml b/ansible/roles/java-base/tasks/main.yml index d84716440..bc1fe57c4 100644 --- a/ansible/roles/java-base/tasks/main.yml +++ b/ansible/roles/java-base/tasks/main.yml @@ -31,6 +31,7 @@ # as appropriate -- try to use generic os family if available. - name: Check if java is already installed + changed_when: no check_mode: no raw: "{{ java_path[os] | default(\"java\") }} -version" register: java diff --git a/ansible/roles/java-base/vars/main.yml b/ansible/roles/java-base/vars/main.yml index cf29e156b..4949f8247 100644 --- a/ansible/roles/java-base/vars/main.yml +++ b/ansible/roles/java-base/vars/main.yml @@ -15,6 +15,7 @@ packages: { 'ibmi': 'openjdk-11-ea', 'macos': 'adoptopenjdk8', 'rhel7': 'java-1.8.0-openjdk', + 'rhel8': 'java-17-openjdk', 'smartos': 'openjdk8', 'ubuntu': 'openjdk-8-jre-headless', 'ubuntu1404': 'oracle-java8-installer', diff --git a/ansible/roles/jenkins-worker/handlers/main.yml b/ansible/roles/jenkins-worker/handlers/main.yml new file mode 100644 index 000000000..6ffdba3ec --- /dev/null +++ b/ansible/roles/jenkins-worker/handlers/main.yml @@ -0,0 +1,15 @@ +--- + +# +# generic handlers for jenkins-worker +# + +- name: restart Jenkins + ansible.builtin.service: + name: jenkins + state: restarted + when: + - not os|startswith("aix") + - not os|startswith("ibmi") + - not os|startswith("macos") + - not os|startswith("zos") diff --git a/ansible/roles/jenkins-worker/tasks/main.yml b/ansible/roles/jenkins-worker/tasks/main.yml index 28ab3f188..8d575bf40 100644 --- a/ansible/roles/jenkins-worker/tasks/main.yml +++ b/ansible/roles/jenkins-worker/tasks/main.yml @@ -131,6 +131,7 @@ mode: 0644 timeout: 60 force: yes + notify: restart Jenkins - name: Resolver | ipnodes needs to be present on zos, similar to hosts file when: os|startswith("zos") @@ -208,6 +209,7 @@ shell: "sh {{ home }}/{{ server_user }}/gen_runAsciiBash.sh" - name: render init script into place + notify: restart Jenkins template: src: "{{ jenkins.src }}" dest: "{{ jenkins.dest }}" @@ -334,12 +336,14 @@ - name: enable jenkins at startup - general + ansible.builtin.service: + name: jenkins + enabled: yes when: - not os|startswith("aix") - not os|startswith("ibmi") - not os|startswith("macos") - not os|startswith("zos") - service: name=jenkins state=restarted enabled=yes - name: Unload org.nodejs.osx.jenkins.plist from launchctl when: os|startswith("macos") diff --git a/ansible/roles/jenkins-worker/vars/main.yml b/ansible/roles/jenkins-worker/vars/main.yml index 9c12d88f4..b9fe350cb 100644 --- a/ansible/roles/jenkins-worker/vars/main.yml +++ b/ansible/roles/jenkins-worker/vars/main.yml @@ -12,7 +12,7 @@ init: { ibmi: 'ibmi73', macos: 'macos', rhel7: 'rhel7', - systemd: ['centos7', 'debian8', 'debian9', 'debian10', 'fedora', 'ubuntu1604', 'ubuntu1804'], + systemd: ['centos7', 'debian8', 'debian9', 'debian10', 'fedora', 'rhel8', 'ubuntu1604', 'ubuntu1804'], svc: 'smartos', upstart: ['ubuntu12', 'ubuntu1404'], zos_start: 'zos' diff --git a/ansible/roles/package-upgrade/vars/main.yml b/ansible/roles/package-upgrade/vars/main.yml index 186cfcb34..592441387 100644 --- a/ansible/roles/package-upgrade/vars/main.yml +++ b/ansible/roles/package-upgrade/vars/main.yml @@ -7,7 +7,7 @@ pm: { 'yum': ['centos', 'rhel7', 'aix', 'ibmi'], 'apt': ['debian', 'ubuntu'], - 'dnf': 'fedora', + 'dnf': ['fedora', 'rhel8'], 'pkg': 'freebsd', 'pkgin': 'smartos', 'chocolatey': 'win', diff --git a/ansible/roles/release-builder/files/ssh_config b/ansible/roles/release-builder/files/ssh_config new file mode 100644 index 000000000..2058e09d2 --- /dev/null +++ b/ansible/roles/release-builder/files/ssh_config @@ -0,0 +1,4 @@ +Host node-www + HostName direct.nodejs.org + User staging + IdentityFile ~/.ssh/id_rsa diff --git a/ansible/roles/release-builder/meta/argument_specs.yml b/ansible/roles/release-builder/meta/argument_specs.yml new file mode 100644 index 000000000..f371432e4 --- /dev/null +++ b/ansible/roles/release-builder/meta/argument_specs.yml @@ -0,0 +1,5 @@ +--- + +argument_specs: + main: + short_description: Set up specific to hosts that build releases. diff --git a/ansible/roles/release-builder/meta/main.yml b/ansible/roles/release-builder/meta/main.yml new file mode 100644 index 000000000..984ee87b7 --- /dev/null +++ b/ansible/roles/release-builder/meta/main.yml @@ -0,0 +1,4 @@ +--- + +dependencies: + - role: user-create diff --git a/ansible/roles/release-builder/tasks/main.yml b/ansible/roles/release-builder/tasks/main.yml new file mode 100644 index 000000000..5fd7e79b2 --- /dev/null +++ b/ansible/roles/release-builder/tasks/main.yml @@ -0,0 +1,42 @@ +--- + +# Set up release hosts to be able to upload to the staging server. +# Requires access to the secrets repository. User should have already +# been prompted for GPG credentials during the inventory load. + +- name: create .ssh directory + ansible.builtin.file: + dest: "{{ home }}/{{ server_user }}/.ssh" + mode: "0700" + owner: "{{ server_user }}" + group: "{{ server_user }}" + state: directory + +- name: copy key to access staging server + ansible.builtin.copy: + content: "{{ lookup('pipe', 'gpg -d ' + staging_key | quote) }}" + dest: "{{ home }}/{{ server_user }}/.ssh/id_rsa" + mode: "0600" + owner: "{{ server_user }}" + group: "{{ server_user }}" + vars: + staging_key: "{{ secrets_repo_root }}/build/release/staging_id_rsa_private.key" + +- name: write ssh_config + ansible.builtin.copy: + dest: "{{ home }}/{{ server_user }}/.ssh/config" + src: "{{ role_path }}/files/ssh_config" + mode: "0600" + owner: "{{ server_user }}" + group: "{{ server_user }}" + +- name: write known_hosts + ansible.builtin.known_hosts: + name: direct.nodejs.org + key: "{{ lookup('pipe', 'gpg -d ' + known_hosts | quote) }}" + path: "{{ home }}/{{ server_user }}/.ssh/known_hosts" + state: present + become: yes + become_user: "{{ server_user }}" + vars: + known_hosts: "{{ secrets_repo_root }}/build/release/known_hosts" diff --git a/ansible/roles/release-builder/vars/main.yml b/ansible/roles/release-builder/vars/main.yml new file mode 100644 index 000000000..fc936ad72 --- /dev/null +++ b/ansible/roles/release-builder/vars/main.yml @@ -0,0 +1,5 @@ +--- + +# The path to the secrets repository. +# See also `ansible/plugins/inventory/nodejs_yaml.py`. +secrets_repo_root: "{{ lookup('env', 'NODE_BUILD_SECRETS')|default('../../../../../secrets', True) }}" diff --git a/jenkins/scripts/VersionSelectorScript.groovy b/jenkins/scripts/VersionSelectorScript.groovy index ba20598b6..cf43db153 100644 --- a/jenkins/scripts/VersionSelectorScript.groovy +++ b/jenkins/scripts/VersionSelectorScript.groovy @@ -30,6 +30,9 @@ def buildExclusions = [ [ /debian8-x86/, anyType, gte(10) ], // 32-bit linux for <10 only [ /debian8/, anyType, gte(13) ], [ /debian9/, anyType, gte(16) ], + [ /rhel7/, releaseType, gte(18) ], + [ /rhel8/, releaseType, lt(18) ], + [ /rhel8/, anyType, lt(14) ], [ /^ubuntu1804/, anyType, lt(10) ], // probably temporary [ /^ubuntu1404-32/, anyType, gte(10) ], // 32-bit linux for <10 only [ /^ubuntu1404-64/, anyType, gte(12) ], diff --git a/jenkins/scripts/select-compiler.sh b/jenkins/scripts/select-compiler.sh index 927c15141..fa976ba8a 100644 --- a/jenkins/scripts/select-compiler.sh +++ b/jenkins/scripts/select-compiler.sh @@ -32,6 +32,29 @@ if [ -z ${NODEJS_MAJOR_VERSION+x} ]; then NODEJS_MAJOR_VERSION="$(echo "$NODE_VERSION" | cut -d . -f 1)" fi +# Linux distros should be arch agnostic +case $NODE_NAME in + *rhel8*) + case "$CONFIG_FLAGS" in + *--enable-lto*) + echo "Setting compiler for Node.js $NODEJS_MAJOR_VERSION (LTO) on" `cat /etc/redhat-release` + . /opt/rh/gcc-toolset-11/enable + export CC="ccache gcc" + export CXX="ccache g++" + echo "Selected compiler:" `${CXX} -dumpversion` + return + ;; + *) + echo "Setting compiler for Node.js $NODEJS_MAJOR_VERSION on" `cat /etc/redhat-release` + # Default gcc on RHEL 8 is gcc 8 + echo "Compiler left as system default:" `g++ -dumpversion` + return + ;; + esac + return + ;; +esac + if [ "$SELECT_ARCH" = "PPC64LE" ]; then # Set default export COMPILER_LEVEL="4.8"