From 920fe6d990bb68eadbe8adfdbfeb5efc0688d8be Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Mon, 3 Feb 2020 15:29:10 +0100 Subject: [PATCH 1/4] Install gcc on staging win2012r2 and ubuntu 18.04 image sets --- imagesets/generic-worker-ubuntu-18-04-staging/bootstrap.sh | 2 +- imagesets/generic-worker-win2012r2-staging/bootstrap.ps1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/imagesets/generic-worker-ubuntu-18-04-staging/bootstrap.sh b/imagesets/generic-worker-ubuntu-18-04-staging/bootstrap.sh index ab4e5e4d..4e4b8030 100644 --- a/imagesets/generic-worker-ubuntu-18-04-staging/bootstrap.sh +++ b/imagesets/generic-worker-ubuntu-18-04-staging/bootstrap.sh @@ -36,7 +36,7 @@ start_time="$(date '+%s')" retry apt update DEBIAN_FRONTEND=noninteractive apt upgrade -yq -retry apt install -y apt-transport-https ca-certificates curl software-properties-common git tar python3-venv python-virtualenv +retry apt install -y apt-transport-https ca-certificates curl software-properties-common git tar python3-venv python-virtualenv build-essential # install docker retry curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - diff --git a/imagesets/generic-worker-win2012r2-staging/bootstrap.ps1 b/imagesets/generic-worker-win2012r2-staging/bootstrap.ps1 index 636ce16f..36851e92 100644 --- a/imagesets/generic-worker-win2012r2-staging/bootstrap.ps1 +++ b/imagesets/generic-worker-win2012r2-staging/bootstrap.ps1 @@ -207,7 +207,7 @@ $client.DownloadFile("https://www.cygwin.com/setup-x86_64.exe", "C:\cygwin-setup # install cygwin # complete package list: https://cygwin.com/packages/package_list.html -Start-Process "C:\cygwin-setup-x86_64.exe" -ArgumentList "--quiet-mode --wait --root C:\cygwin --site http://cygwin.mirror.constant.com --packages openssh,vim,curl,tar,wget,zip,unzip,diffutils,bzr" -Wait -NoNewWindow -PassThru -RedirectStandardOutput "C:\cygwin_install.log" -RedirectStandardError "C:\cygwin_install.err" +Start-Process "C:\cygwin-setup-x86_64.exe" -ArgumentList "--quiet-mode --wait --root C:\cygwin --site http://cygwin.mirror.constant.com --packages openssh,vim,curl,tar,wget,zip,unzip,diffutils,bzr,gcc-g++" -Wait -NoNewWindow -PassThru -RedirectStandardOutput "C:\cygwin_install.log" -RedirectStandardError "C:\cygwin_install.err" # open up firewall for ssh daemon New-NetFirewallRule -DisplayName "Allow SSH inbound" -Direction Inbound -LocalPort 22 -Protocol TCP -Action Allow From 24efc2ad87ddae24dec14ba754d443079b1fb2d9 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Tue, 4 Feb 2020 15:10:33 +0100 Subject: [PATCH 2/4] wip --- config/imagesets.yml | 11 +++--- config/projects.yml | 2 +- .../bootstrap.sh | 39 +++++++++++-------- 3 files changed, 29 insertions(+), 23 deletions(-) diff --git a/config/imagesets.yml b/config/imagesets.yml index e1756d52..59e09d75 100644 --- a/config/imagesets.yml +++ b/config/imagesets.yml @@ -74,11 +74,12 @@ generic-worker-win2012r2-staging: workerImplementation: generic-worker aws: amis: - us-east-1: ami-0162f92052875bf7e - us-west-1: ami-0f6b26130f2f9ce2c - us-west-2: ami-0dfd8045878e6cd08 - gcp: - image: projects/community-tc-workers/global/images/generic-worker-win2012r2-staging-hasr18iepqtggvlswlku + us-east-1: ami-0f892d6ff7cdc52e7 + us-west-1: ami-0cf711badfa2f50fc + us-west-2: ami-02ed2a69fdfdef0bc +# Currently not deployed in gcp... +# gcp: +# image: projects/community-tc-workers/global/images/generic-worker-win2012r2-staging-hasr18iepqtggvlswlku workerConfig: genericWorker: config: diff --git a/config/projects.yml b/config/projects.yml index 7aee3638..149a2c7f 100644 --- a/config/projects.yml +++ b/config/projects.yml @@ -152,7 +152,7 @@ taskcluster: owner: taskcluster-notifications+workers@mozilla.com emailOnError: true imageset: generic-worker-win2012r2-staging - cloud: gcp + cloud: aws maxCapacity: 10 workerConfig: genericWorker: diff --git a/imagesets/generic-worker-ubuntu-18-04-staging/bootstrap.sh b/imagesets/generic-worker-ubuntu-18-04-staging/bootstrap.sh index 4e4b8030..fcd0e15f 100644 --- a/imagesets/generic-worker-ubuntu-18-04-staging/bootstrap.sh +++ b/imagesets/generic-worker-ubuntu-18-04-staging/bootstrap.sh @@ -5,9 +5,10 @@ exec &> /var/log/bootstrap.log # Version numbers #################### WORKER_RUNNER_VERSION='v1.0.1' +# GENERIC_WORKER_REF='v16.5.6' +GENERIC_WORKER_VERSION='v16.5.6' LIVELOG_VERSION='v1.1.0' TASKCLUSTER_PROXY_VERSION='v5.1.0' -GENERIC_WORKER_REF='v16.5.6' ###################################### function retry { @@ -36,7 +37,8 @@ start_time="$(date '+%s')" retry apt update DEBIAN_FRONTEND=noninteractive apt upgrade -yq -retry apt install -y apt-transport-https ca-certificates curl software-properties-common git tar python3-venv python-virtualenv build-essential +# retry apt install -y apt-transport-https ca-certificates curl software-properties-common git tar python3-venv python-virtualenv build-essential +retry apt install -y apt-transport-https ca-certificates curl software-properties-common gzip python3-venv python-virtualenv build-essential # install docker retry curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - @@ -48,25 +50,28 @@ sleep 5 systemctl status docker | grep "Started Docker Application Container Engine" usermod -aG docker ubuntu -# build generic-worker from ${GENERIC_WORKER_REF} commit / branch / tag etc -retry curl -L 'https://dl.google.com/go/go1.12.9.linux-amd64.tar.gz' > go.tar.gz -tar xvfz go.tar.gz -C /usr/local -export HOME=/root -export GOPATH=~/go -export GOROOT=/usr/local/go -export PATH="${GOROOT}/bin:${GOPATH}/bin:${PATH}" -go get -d github.com/taskcluster/generic-worker -cd "${GOPATH}/src/github.com/taskcluster/generic-worker" -git checkout "${GENERIC_WORKER_REF}" -CGO_ENABLED=0 go install -tags multiuser -ldflags "-X main.revision=$(git rev-parse HEAD)" github.com/taskcluster/generic-worker -mv "${GOPATH}/bin/generic-worker" /usr/local/bin/ - -# install livelog and taskcluster-proxy +# # build generic-worker from ${GENERIC_WORKER_REF} commit / branch / tag etc +# retry curl -L 'https://dl.google.com/go/go1.12.9.linux-amd64.tar.gz' > go.tar.gz +# tar xvfz go.tar.gz -C /usr/local +# export HOME=/root +# export GOPATH=~/go +# export GOROOT=/usr/local/go +# export PATH="${GOROOT}/bin:${GOPATH}/bin:${PATH}" +# go get -d github.com/taskcluster/generic-worker +# cd "${GOPATH}/src/github.com/taskcluster/generic-worker" +# git checkout "${GENERIC_WORKER_REF}" +# CGO_ENABLED=0 go install -tags multiuser -ldflags "-X main.revision=$(git rev-parse HEAD)" github.com/taskcluster/generic-worker +# mv "${GOPATH}/bin/generic-worker" /usr/local/bin/ +# +# # install livelog and taskcluster-proxy cd /usr/local/bin +# next line can be removed if building generic-worker... +retry curl -L "https://github.com/taskcluster/generic-worker/releases/download/${GENERIC_WORKER_VERSION}/generic-worker-multiuser-linux-amd64" > generic-worker retry curl -L "https://github.com/taskcluster/taskcluster-worker-runner/releases/download/${WORKER_RUNNER_VERSION}/start-worker-linux-amd64" > start-worker retry curl -L "https://github.com/taskcluster/livelog/releases/download/${LIVELOG_VERSION}/livelog-linux-amd64" > livelog retry curl -L "https://github.com/taskcluster/taskcluster-proxy/releases/download/${TASKCLUSTER_PROXY_VERSION}/taskcluster-proxy-linux-amd64" > taskcluster-proxy -chmod a+x start-worker taskcluster-proxy livelog +# chmod a+x start-worker taskcluster-proxy livelog +chmod a+x generic-worker start-worker taskcluster-proxy livelog mkdir -p /etc/generic-worker mkdir -p /var/local/generic-worker From b5e9d2c58bc795cb089104750f5f3c9aa8af6ae7 Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Tue, 4 Feb 2020 15:29:11 +0100 Subject: [PATCH 3/4] Empty commit to avoid reusing commit sha From 5a2e431633dbb1c7089ceae9bb4baa3cd9e81bcd Mon Sep 17 00:00:00 2001 From: Pete Moore Date: Tue, 4 Feb 2020 15:47:53 +0100 Subject: [PATCH 4/4] wip --- config/imagesets.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/imagesets.yml b/config/imagesets.yml index 59e09d75..bf23ebdc 100644 --- a/config/imagesets.yml +++ b/config/imagesets.yml @@ -134,9 +134,9 @@ generic-worker-ubuntu-18-04-staging: workerImplementation: generic-worker aws: amis: - us-east-1: ami-06ce1a5f3039ac646 - us-west-1: ami-0bd5b1b63c47a7b65 - us-west-2: ami-0a396cff215e28640 + us-east-1: ami-0e785d8c0d91ba485 + us-west-1: ami-03b04649c13e7ec0c + us-west-2: ami-021550227b98d65e7 workerConfig: genericWorker: config: