From d67e31c9d93aab66cf3e080e56351287eafcb17e Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Fri, 2 Feb 2024 18:20:36 -0700 Subject: [PATCH 01/46] python and python3 --- roles/gitian/templates/gitian-parallel-build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/gitian/templates/gitian-parallel-build.sh b/roles/gitian/templates/gitian-parallel-build.sh index fd87c6e..3051cac 100644 --- a/roles/gitian/templates/gitian-parallel-build.sh +++ b/roles/gitian/templates/gitian-parallel-build.sh @@ -32,6 +32,9 @@ zcash_binaries_dir_path=${HOME}/zcash-binaries build_dir_path=${gitian_builder_repo_path}/build suite_descriptors_dir_path=${gitian_builder_repo_path}/suites +#workaround python and python3 +ln -s $(which python3) $(which python3 | sed 's/3//g') + # Help Message read -d '' usage <<- EOF Usage: $scriptName [-c|u|v|b|o|h|j|m|] signer version From 7cbd8c67a7bfc0f81e206c5f2c39fd28677bf6df Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Fri, 2 Feb 2024 19:16:55 -0700 Subject: [PATCH 02/46] Update gitian-parallel-build.sh --- roles/gitian/templates/gitian-parallel-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/gitian/templates/gitian-parallel-build.sh b/roles/gitian/templates/gitian-parallel-build.sh index 3051cac..d308d9f 100644 --- a/roles/gitian/templates/gitian-parallel-build.sh +++ b/roles/gitian/templates/gitian-parallel-build.sh @@ -33,7 +33,7 @@ build_dir_path=${gitian_builder_repo_path}/build suite_descriptors_dir_path=${gitian_builder_repo_path}/suites #workaround python and python3 -ln -s $(which python3) $(which python3 | sed 's/3//g') +sudo ln -s $(which python3) $(which python3 | sed 's/3//g') # Help Message read -d '' usage <<- EOF From 20559e78d2bc0625be43a298492e7b8134e236f2 Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Fri, 9 Feb 2024 12:06:31 -0700 Subject: [PATCH 03/46] ADD CI --- .github/workflows/CI.yaml | 251 ++++++++++++++++++ roles/gitian/templates/gitian-build.sh | 6 +- .../gitian/templates/gitian-parallel-build.sh | 6 +- 3 files changed, 261 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/CI.yaml diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml new file mode 100644 index 0000000..bfa592a --- /dev/null +++ b/.github/workflows/CI.yaml @@ -0,0 +1,251 @@ +name: Get Label PR + +on: + pull_request: + types: + - labeled + +jobs: + obtener-label: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v2' + with: + version: '>= 363.0.0' + + - name: Get Label PR + id: obtener-label + run: | + sudo apt update; sudo apt install wget openssh-client git -y + echo ${{ secrets.GCP_SA_KEY }} | base64 -d > json.json + gcloud auth activate-service-account --key-file=json.json + + export random=$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 4; echo;) + + for i in $(gcloud compute os-login ssh-keys list --format="table[no-heading](value.fingerprint)"); do + echo $i; + gcloud compute os-login ssh-keys remove --key $i || true; + done + + gcloud compute instances create test-gitian-$random --image-family=debian-11 --image-project=debian-cloud --machine-type=c2-standard-16 --project=${{ secrets.GCP_PROJECT_ID_PROD }}' --zone=us-central1-a --no-address --network=vpc-${{ secrets.GCP_PROJECT_ID_PROD }}' --subnet=us-central1-zcash --tags=zcash --service-account=vm-iap@${{ secrets.GCP_PROJECT_ID_PROD }}'.iam.gserviceaccount.com --metadata=enable-oslogin=TRUE --scopes=cloud-platform --enable-nested-virtualization --boot-disk-size=200GB + + export counter=1 + while [[ $(gcloud compute ssh --zone "us-central1-a" "test-gitian-$random" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}'" --command="ls -la" &>/dev/null || echo "re-try") == "re-try" && counter -lt 60 ]] + do + echo "attemp number: $counter" + export counter=$((counter+1)) + if [ $counter -eq 60 ]; then gcloud compute instances delete "test-gitian-$random" --project "${{ secrets.GCP_PROJECT_ID_PROD }}'" --zone "us-central1-a" --delete-disks=all; exit 1; fi + sleep 5 + done + + IFS='/' read -r -a array <<< "${{ github.event.label.name }}" + + git clone -b ${array[2]} https://github.com/${array[0]}/${array[1]}.git + cd zcash/contrib/gitian-descriptors + wget -c https://github.com/mikefarah/yq/releases/download/v4.28.2/yq_linux_amd64 + chmod +x yq_linux_amd64 + export ZCASH_GITIAN_VERSION=$(cat gitian-linux-parallel.yml | ./yq_linux_amd64 .name) + cd ../../.. + + + cat < ./script.sh + apt update; + apt install ca-certificates curl gnupg lsb-release zsh software-properties-common wget git vagrant python3-venv direnv python3-pip linux-headers-\$(uname -r) ansible -y; + mkdir -m 0755 -p /etc/apt/keyrings; + curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg --yes; + echo "deb [arch=\$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \$(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + apt update; + apt install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y; + apt-add-repository "deb http://download.virtualbox.org/virtualbox/debian \$(lsb_release -sc) contrib"; + wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc -O- | apt-key add -; + apt update + apt install virtualbox-6.1 -y; + eval "\$(direnv hook bash)"; + cd source + cp .env.example .env + cp .envrc.example .envrc + /usr/bin/python3 -m venv ./local/python_venv; + echo "load_prefix local/python_venv" >> .envrc; + export VERSION="${array[2]}" + echo "ZCASH_VERSION=\$VERSION" >> .env; + echo "ZCASH_GIT_REPO_URL=https://github.com/${array[0]}/${array[1]}" >> .env; + cat .env + direnv allow; + pip3 install --upgrade pip; + /sbin/vboxconfig; + vagrant plugin install --local; + vagrant plugin install --local; + gpg --quick-generate-key --batch --passphrase '' "Harry Potter (zcash gitian) " + echo "GPG_KEY_ID=\$(gpg --list-keys --with-fingerprint --with-colons | grep fpr: | head -n 1 | sed 's/fpr://g' | sed 's/://g')" >> .env; + git config --global user.name "Harry Potter" + git config --global user.email "hpotter@hogwarts.wiz" + direnv allow; + direnv exec \$(pwd) vagrant up zcash-build; + vagrant ssh zcash-build -c "gpg --quick-generate-key --batch --passphrase '' \"Harry Potter (zcash gitian) \" || echo ''" + vagrant ssh zcash-build -c "./gitian-parallel-build.sh || exit 1" + vagrant ssh zcash-build -c "head -n 8 gitian.sigs/\$VERSION*/hpotter/*.assert" > assert.txt + tr -d \$'\r' < assert.txt > assert2.txt + for i in \$(cat assert2.txt | grep -E "zcash-*" | grep -v git: | sed 's/ //g' | sed 's/ /-->/g'); do + curl ${{ secrets.SLACK_WEBHOOK }} -H "Content-Type: application/json" -d "{\"text\": \"\\\`\\\`\\\`\$i\\\`\\\`\\\`\"}" + done + export OS=\$(vagrant ssh zcash-build -c "ls zcash-binaries/\$VERSION" | tr -d '\r') + for i in \$OS; do vagrant ssh zcash-build -c "mkdir \$i; tar Cxvzf \$i zcash-binaries/*/\$i/zcash-*-linux64.tar.gz"; done + versions=\$(for i in \$OS; do echo " \$i==>"; vagrant ssh zcash-build -c "./\$i/zcash-*/bin/zcashd --version | head -n 1 | tr -d '\n'"; done) + for i in "\${versions[@]}" + do + curl ${{ secrets.SLACK_WEBHOOK }} -H "Content-Type: application/json" -d "{\"text\": \"\\\`\\\`\\\`\$i\\\`\\\`\\\`\"}" + done + + # get keys + gsutil rm -r gs://${{ secrets.GCP_PROJECT_ID_PROD }}'-apt-packages/127.0.0.1 || echo "" + gsutil cp gs://${{ secrets.GCP_PROJECT_ID_PROD }}'-apt-packages/encrypted_gpg.kms \$HOME/encrypted_gpg.kms + gsutil cp gs://${{ secrets.GCP_PROJECT_ID_PROD }}'-apt-packages/public.asc \$HOME/public.asc + current_dir=\$(pwd) + cd \$HOME + gcloud kms decrypt \ + --key gpg \ + --keyring gpg \ + --location global \ + --plaintext-file private.pgp \ + --ciphertext-file encrypted_gpg.kms + cd \$current_dir + gpg --import \$HOME/private.pgp + vagrant scp :gitian.sigs . + for i in \$OS; + do + mkdir -p debs/\$i; + mkdir -p ./\$i-extract + vagrant ssh zcash-build -c "mkdir /home/vagrant/"\$i"-extract"; + vagrant ssh zcash-build -c "tar -xvf /home/vagrant/zcash-binaries/"\$VERSION"/"\$i"/zcash-*-linux64.tar.gz -C /home/vagrant/"\$i"-extract"; + + docker run -d --name \$i debian:\$i bash -c "while true; do sleep 2; done"; + + docker exec \$i bash -c "mkdir -p /home/vagrant/\$i-deb-build && cd /home/vagrant/\$i-deb-build && apt update && apt install git dpkg-dev lintian -y && git clone -b ${array[2]} https://github.com/${array[0]}/${array[1]}.git ."; + + vagrant scp :/home/vagrant/\$i-extract/zcash-*/bin/zcash-tx ./\$i-extract/ + vagrant scp :/home/vagrant/\$i-extract/zcash-*/bin/zcash-fetch-params ./\$i-extract/ + vagrant scp :/home/vagrant/\$i-extract/zcash-*/bin/zcashd ./\$i-extract/ + vagrant scp :/home/vagrant/\$i-extract/zcash-*/bin/zcash-cli ./\$i-extract/ + vagrant scp :/home/vagrant/\$i-extract/zcash-*/bin/zcashd-wallet-tool ./\$i-extract/ + docker cp ./\$i-extract \$i:/home/vagrant/\$i-deb-build/ + docker exec -w /home/vagrant/\$i-deb-build \$i bash -c "rm -rf src && mv \$i-extract src && ./zcutil/build-debian-package.sh" + docker cp \$i:/tmp/zcbuild ./debs/\$i + done + vagrant scp :/home/vagrant/zcash-binaries ./ + for i in \$OS; + do + cd ./zcash-binaries/\$VERSION/\$i + for j in \$(ls *linux64.tar.gz); do + mv \$j \$(echo \$j | sed 's/.tar.gz/-debian-'\$i'.tar.gz/g') + done + for j in \$(ls *debug.tar.gz); do + mv \$j \$(echo \$j | sed 's/.tar.gz/-debian-'\$i'.tar.gz/g') + done + gpg -u sysadmin@z.cash --armor --digest-algo SHA256 --detach-sign *debug-debian-\$i.tar.gz + gpg -u sysadmin@z.cash --armor --digest-algo SHA256 --detach-sign *linux64-debian-\$i.tar.gz + cd \$current_dir + done + export final_version=\$(cat assert2.txt | awk '{print \$2}' | grep "desc.yml" | head -n 1 | sed 's/-desc.yml//g') + gsutil -m rsync -r ./debs gs://${{ secrets.GCP_PROJECT_ID_PROD }}'-apt-packages/debs + gsutil -m rsync -r ./zcash-binaries gs://${{ secrets.GCP_PROJECT_ID_PROD }}'-apt-packages/zcash-binaries + apt install aptly -y + + # generate apt + mkdir aptserver + cd aptserver + gsutil -m cp -r gs://${{ secrets.GCP_PROJECT_ID_PROD }}'-apt-server/pool/main/z/zcash/ . + cd zcash + cp -a ../../debs/buster/zcbuild/*.deb \$final_version-amd64-buster.deb + ls \$final_version-amd64-buster.deb || exit 1 + cp -a ../../debs/bullseye/zcbuild/*.deb \$final_version-amd64-bullseye.deb + ls \$final_version-amd64-bullseye.deb || exit 1 + cp -a ../../debs/bookworm/zcbuild/*.deb \$final_version-amd64-bookworm.deb + ls \$final_version-amd64-bookworm.deb || exit 1 + aptly repo create --distribution buster --comment "" --component main zcash_buster_amd64_repo + aptly repo create --distribution bullseye --comment "" --component main zcash_bullseye_amd64_repo + aptly repo create --distribution bookworm --comment "" --component main zcash_bookworm_amd64_repo + aptly repo create --distribution stretch --comment "" --component main zcash_stretch_amd64_repo + for i in \$(ls *.deb | grep buster); do + aptly repo add zcash_buster_amd64_repo \$i + done + for i in \$(ls *.deb | grep bullseye); do + aptly repo add zcash_bullseye_amd64_repo \$i + done + for i in \$(ls *.deb | grep stretch); do + aptly repo add zcash_stretch_amd64_repo \$i + done + for i in \$(ls *.deb | grep bookworm); do + aptly repo add zcash_bookworm_amd64_repo \$i + done + aptly snapshot create bookworm_snapshot from repo zcash_bookworm_amd64_repo + aptly snapshot create buster_snapshot from repo zcash_buster_amd64_repo + aptly snapshot create bullseye_snapshot from repo zcash_bullseye_amd64_repo + aptly snapshot create stretch_snapshot from repo zcash_stretch_amd64_repo + + export key=\$(gpg --list-secret-keys --keyid-format=long sysadmin@z.cash | head -n 2 | grep -v sec) + aptly publish snapshot --distribution buster --component main --architectures amd64 --gpg-key="\$key" --passphrase="" buster_snapshot + aptly publish snapshot --distribution bookworm --component main --architectures amd64 --gpg-key="\$key" --passphrase="" bookworm_snapshot + aptly publish snapshot --distribution bullseye --component main --architectures amd64 --gpg-key="\$key" --passphrase="" bullseye_snapshot + aptly publish snapshot --distribution stretch --component main --architectures amd64 --gpg-key="\$key" --passphrase="" stretch_snapshot + + apt install nginx-extras -y + cat << EOH > /etc/nginx/sites-enabled/default + server { + listen 80 default_server; + root /var/www/public; + location / { + autoindex on; + } + server_name _; + } + EOH + # get apt server + cp -a /root/.aptly/public /var/www/ + chown -R www-data:www-data /var/www + /etc/init.d/nginx restart + mkdir \$HOME/mirror + cd \$HOME/mirror + wget -r 127.0.0.1 + + cp \$HOME/public.asc \$HOME/mirror/127.0.0.1/zcash.asc + cd \$HOME/mirror + gsutil -m rsync -r ./127.0.0.1 gs://${{ secrets.GCP_PROJECT_ID_PROD }}'-apt-packages/127.0.0.1 + cd 127.0.0.1 + if ! [[ ${array[2]} == *"-rc"* ]]; then + gsutil -m rsync -r ./ gs://${{ secrets.GCP_PROJECT_ID_PROD }}'-apt-server/ + fi + EOF + + export FAIL=0 + chmod +x ./script.sh || echo "" + + gcloud compute scp ./script.sh --zone "us-central1-a" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}'" test-gitian-$random: || export FAIL=1 + gcloud compute scp --recurse $(pwd) --zone "us-central1-a" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}'" test-gitian-$random:~/source || export FAIL=1 + + gcloud compute ssh --zone "us-central1-a" "test-gitian-$random" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}'" --command="bash -i -c 'sudo -s ./script.sh'" -- -t || export FAIL=1 + + gcloud compute instances delete "test-gitian-$random" --project "${{ secrets.GCP_PROJECT_ID_PROD }}'" --zone "us-central1-a" --delete-disks=all + + if [ $FAIL -eq 1 ]; then exit 1; fi + + gcloud compute scp --recurse --zone "us-central1-a" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}'" test-gitian-$random:/home/sa_*/source/gitian.sigs . + + rm -rf gitian.sigs/.git + if ! [[ ${array[2]} == *"-rc"* ]]; then + git clone git@github.com:zcash/gitian.sigs.git sigs + cp -a gitian.sigs/* sigs/ + cd sigs + git config --global user.name "ECC-CI" + git config --global user.email "serviceusers+github@z.cash" + git add . + git commit -am "$(inputs.params.LABEL_NAME)" + git push + fi + curl --request POST --url https://api.bunny.net/pullzone/1432616/purgeCache --header 'content-type: application/json' --header 'AccessKey: ${{ secrets.BUNNY_API_KEY }}' + + shell: bash diff --git a/roles/gitian/templates/gitian-build.sh b/roles/gitian/templates/gitian-build.sh index dafca5c..b60fe34 100644 --- a/roles/gitian/templates/gitian-build.sh +++ b/roles/gitian/templates/gitian-build.sh @@ -215,7 +215,11 @@ then echo "" echo "Compiling variant: ${VERSION}_${suite}" echo "" - + #workaround python and python3 in buster + if [[ $suite = "buster" ]] + then + sed -i -e 's/- "python3"/- "python"/g' -e 's/- "python-is-python3"//g' ${suite_dir_path}/gitian-linux-parallel.yml; + fi ./bin/gbuild --fetch-tags -j ${proc} -m ${mem} --commit zcash=${COMMIT} --url zcash=${url} ${suite_dir_path}/gitian-linux.yml ./bin/gsign -p "$signProg" --signer "$SIGNER" --release ${VERSION}_${suite} --destination ${gitian_sigs_repo_path}/ ${suite_dir_path}/gitian-linux.yml diff --git a/roles/gitian/templates/gitian-parallel-build.sh b/roles/gitian/templates/gitian-parallel-build.sh index fd87c6e..5600436 100644 --- a/roles/gitian/templates/gitian-parallel-build.sh +++ b/roles/gitian/templates/gitian-parallel-build.sh @@ -215,7 +215,11 @@ then echo "" echo "Compiling variant: ${VERSION}_${suite}" echo "" - + #workaround python and python3 in buster + if [[ $suite = "buster" ]] + then + sed -i -e 's/- "python3"/- "python"/g' -e 's/- "python-is-python3"//g' ${suite_dir_path}/gitian-linux-parallel.yml; + fi ./bin/gbuild --fetch-tags -j ${proc} -m ${mem} --commit zcash=${COMMIT} --url zcash=${url} ${suite_dir_path}/gitian-linux-parallel.yml ./bin/gsign -p "$signProg" --signer "$SIGNER" --release ${VERSION}_${suite} --destination ${gitian_sigs_repo_path}/ ${suite_dir_path}/gitian-linux-parallel.yml From fc13346f8a516b04f801fee6809e6afabef5837a Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Fri, 9 Feb 2024 12:08:25 -0700 Subject: [PATCH 04/46] Update gitian-parallel-build.sh --- roles/gitian/templates/gitian-parallel-build.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/roles/gitian/templates/gitian-parallel-build.sh b/roles/gitian/templates/gitian-parallel-build.sh index aa0e726..5600436 100644 --- a/roles/gitian/templates/gitian-parallel-build.sh +++ b/roles/gitian/templates/gitian-parallel-build.sh @@ -32,9 +32,6 @@ zcash_binaries_dir_path=${HOME}/zcash-binaries build_dir_path=${gitian_builder_repo_path}/build suite_descriptors_dir_path=${gitian_builder_repo_path}/suites -#workaround python and python3 -sudo ln -s $(which python3) $(which python3 | sed 's/3//g') - # Help Message read -d '' usage <<- EOF Usage: $scriptName [-c|u|v|b|o|h|j|m|] signer version From fa67e9199bf68195bc2148b19a3a844b497a29e1 Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Fri, 9 Feb 2024 13:45:56 -0700 Subject: [PATCH 05/46] Update CI.yaml --- .github/workflows/CI.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index bfa592a..f1af36a 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -18,8 +18,8 @@ jobs: with: version: '>= 363.0.0' - - name: Get Label PR - id: obtener-label + - name: Build Gitian + id: gitian run: | sudo apt update; sudo apt install wget openssh-client git -y echo ${{ secrets.GCP_SA_KEY }} | base64 -d > json.json From aea574e47aacecf58648507f48f7e06823b69249 Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Fri, 9 Feb 2024 13:56:55 -0700 Subject: [PATCH 06/46] Update CI.yaml --- .github/workflows/CI.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index f1af36a..2e0d0b9 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -1,4 +1,4 @@ -name: Get Label PR +name: Gitian CI on: pull_request: From 0c2b2414a1197ad489402f3d0b07be61530f9812 Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Fri, 9 Feb 2024 14:02:36 -0700 Subject: [PATCH 07/46] Update CI.yaml --- .github/workflows/CI.yaml | 49 +++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 2e0d0b9..68ee2fa 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -27,19 +27,19 @@ jobs: export random=$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 4; echo;) - for i in $(gcloud compute os-login ssh-keys list --format="table[no-heading](value.fingerprint)"); do - echo $i; + for i in $(gcloud compute os-login ssh-keys list --format="table[no-heading](value.fingerprint)"); do + echo $i; gcloud compute os-login ssh-keys remove --key $i || true; done - gcloud compute instances create test-gitian-$random --image-family=debian-11 --image-project=debian-cloud --machine-type=c2-standard-16 --project=${{ secrets.GCP_PROJECT_ID_PROD }}' --zone=us-central1-a --no-address --network=vpc-${{ secrets.GCP_PROJECT_ID_PROD }}' --subnet=us-central1-zcash --tags=zcash --service-account=vm-iap@${{ secrets.GCP_PROJECT_ID_PROD }}'.iam.gserviceaccount.com --metadata=enable-oslogin=TRUE --scopes=cloud-platform --enable-nested-virtualization --boot-disk-size=200GB + gcloud compute instances create test-gitian-$random --image-family=debian-11 --image-project=debian-cloud --machine-type=c2-standard-16 --project=ecc-infra-prod --zone=us-central1-a --no-address --network=vpc-ecc-infra-prod --subnet=us-central1-zcash --tags=zcash --service-account=vm-iap@ecc-infra-prod.iam.gserviceaccount.com --metadata=enable-oslogin=TRUE --scopes=cloud-platform --enable-nested-virtualization --boot-disk-size=200GB export counter=1 - while [[ $(gcloud compute ssh --zone "us-central1-a" "test-gitian-$random" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}'" --command="ls -la" &>/dev/null || echo "re-try") == "re-try" && counter -lt 60 ]] + while [[ $(gcloud compute ssh --zone "us-central1-a" "test-gitian-$random" --tunnel-through-iap --project "ecc-infra-prod" --command="ls -la" &>/dev/null || echo "re-try") == "re-try" && counter -lt 60 ]] do echo "attemp number: $counter" export counter=$((counter+1)) - if [ $counter -eq 60 ]; then gcloud compute instances delete "test-gitian-$random" --project "${{ secrets.GCP_PROJECT_ID_PROD }}'" --zone "us-central1-a" --delete-disks=all; exit 1; fi + if [ $counter -eq 60 ]; then gcloud compute instances delete "test-gitian-$random" --project "ecc-infra-prod" --zone "us-central1-a" --delete-disks=all; exit 1; fi sleep 5 done @@ -87,7 +87,7 @@ jobs: direnv allow; direnv exec \$(pwd) vagrant up zcash-build; vagrant ssh zcash-build -c "gpg --quick-generate-key --batch --passphrase '' \"Harry Potter (zcash gitian) \" || echo ''" - vagrant ssh zcash-build -c "./gitian-parallel-build.sh || exit 1" + vagrant ssh zcash-build -c ./gitian-parallel-build.sh || exit 1 vagrant ssh zcash-build -c "head -n 8 gitian.sigs/\$VERSION*/hpotter/*.assert" > assert.txt tr -d \$'\r' < assert.txt > assert2.txt for i in \$(cat assert2.txt | grep -E "zcash-*" | grep -v git: | sed 's/ //g' | sed 's/ /-->/g'); do @@ -102,9 +102,9 @@ jobs: done # get keys - gsutil rm -r gs://${{ secrets.GCP_PROJECT_ID_PROD }}'-apt-packages/127.0.0.1 || echo "" - gsutil cp gs://${{ secrets.GCP_PROJECT_ID_PROD }}'-apt-packages/encrypted_gpg.kms \$HOME/encrypted_gpg.kms - gsutil cp gs://${{ secrets.GCP_PROJECT_ID_PROD }}'-apt-packages/public.asc \$HOME/public.asc + gsutil rm -r gs://ecc-infra-prod-apt-packages/127.0.0.1 || echo "" + gsutil cp gs://ecc-infra-prod-apt-packages/encrypted_gpg.kms \$HOME/encrypted_gpg.kms + gsutil cp gs://ecc-infra-prod-apt-packages/public.asc \$HOME/public.asc current_dir=\$(pwd) cd \$HOME gcloud kms decrypt \ @@ -151,21 +151,23 @@ jobs: cd \$current_dir done export final_version=\$(cat assert2.txt | awk '{print \$2}' | grep "desc.yml" | head -n 1 | sed 's/-desc.yml//g') - gsutil -m rsync -r ./debs gs://${{ secrets.GCP_PROJECT_ID_PROD }}'-apt-packages/debs - gsutil -m rsync -r ./zcash-binaries gs://${{ secrets.GCP_PROJECT_ID_PROD }}'-apt-packages/zcash-binaries + gsutil -m rsync -r ./debs gs://ecc-infra-prod-apt-packages/debs + gsutil -m rsync -r ./zcash-binaries gs://ecc-infra-prod-apt-packages/zcash-binaries apt install aptly -y # generate apt mkdir aptserver cd aptserver - gsutil -m cp -r gs://${{ secrets.GCP_PROJECT_ID_PROD }}'-apt-server/pool/main/z/zcash/ . + gsutil -m cp -r gs://ecc-infra-prod-apt-server/pool/main/z/zcash/ . cd zcash cp -a ../../debs/buster/zcbuild/*.deb \$final_version-amd64-buster.deb - ls \$final_version-amd64-buster.deb || exit 1 cp -a ../../debs/bullseye/zcbuild/*.deb \$final_version-amd64-bullseye.deb - ls \$final_version-amd64-bullseye.deb || exit 1 cp -a ../../debs/bookworm/zcbuild/*.deb \$final_version-amd64-bookworm.deb + + ls \$final_version-amd64-buster.deb || exit 1 + ls \$final_version-amd64-bullseye.deb || exit 1 ls \$final_version-amd64-bookworm.deb || exit 1 + aptly repo create --distribution buster --comment "" --component main zcash_buster_amd64_repo aptly repo create --distribution bullseye --comment "" --component main zcash_bullseye_amd64_repo aptly repo create --distribution bookworm --comment "" --component main zcash_bookworm_amd64_repo @@ -214,26 +216,26 @@ jobs: cp \$HOME/public.asc \$HOME/mirror/127.0.0.1/zcash.asc cd \$HOME/mirror - gsutil -m rsync -r ./127.0.0.1 gs://${{ secrets.GCP_PROJECT_ID_PROD }}'-apt-packages/127.0.0.1 + gsutil -m rsync -r ./127.0.0.1 gs://ecc-infra-prod-apt-packages/127.0.0.1 cd 127.0.0.1 if ! [[ ${array[2]} == *"-rc"* ]]; then - gsutil -m rsync -r ./ gs://${{ secrets.GCP_PROJECT_ID_PROD }}'-apt-server/ + gsutil -m rsync -r ./ gs://ecc-infra-prod-apt-server/ fi EOF export FAIL=0 chmod +x ./script.sh || echo "" - gcloud compute scp ./script.sh --zone "us-central1-a" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}'" test-gitian-$random: || export FAIL=1 - gcloud compute scp --recurse $(pwd) --zone "us-central1-a" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}'" test-gitian-$random:~/source || export FAIL=1 + gcloud compute scp ./script.sh --zone "us-central1-a" --tunnel-through-iap --project "ecc-infra-prod" test-gitian-$random: || export FAIL=1 + gcloud compute scp --recurse $(pwd) --zone "us-central1-a" --tunnel-through-iap --project "ecc-infra-prod" test-gitian-$random:~/source || export FAIL=1 - gcloud compute ssh --zone "us-central1-a" "test-gitian-$random" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}'" --command="bash -i -c 'sudo -s ./script.sh'" -- -t || export FAIL=1 + gcloud compute ssh --zone "us-central1-a" "test-gitian-$random" --tunnel-through-iap --project "ecc-infra-prod" --command="bash -i -c 'sudo -s ./script.sh'" -- -t || export FAIL=1 - gcloud compute instances delete "test-gitian-$random" --project "${{ secrets.GCP_PROJECT_ID_PROD }}'" --zone "us-central1-a" --delete-disks=all + gcloud compute scp --recurse --zone "us-central1-a" --tunnel-through-iap --project "ecc-infra-prod" test-gitian-$random:/home/sa_*/source/gitian.sigs . + + gcloud compute instances delete "test-gitian-$random" --project "ecc-infra-prod" --zone "us-central1-a" --delete-disks=all - if [ $FAIL -eq 1 ]; then exit 1; fi - - gcloud compute scp --recurse --zone "us-central1-a" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}'" test-gitian-$random:/home/sa_*/source/gitian.sigs . + if [ $FAIL -eq 1 ]; then exit 1; fi rm -rf gitian.sigs/.git if ! [[ ${array[2]} == *"-rc"* ]]; then @@ -246,6 +248,7 @@ jobs: git commit -am "$(inputs.params.LABEL_NAME)" git push fi + curl --request POST --url https://api.bunny.net/pullzone/1432616/purgeCache --header 'content-type: application/json' --header 'AccessKey: ${{ secrets.BUNNY_API_KEY }}' shell: bash From a188399711c482318fabd39f96d7477613abaec3 Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Fri, 9 Feb 2024 14:48:44 -0700 Subject: [PATCH 08/46] Update CI.yaml --- .github/workflows/CI.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 68ee2fa..08f000d 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -6,7 +6,7 @@ on: - labeled jobs: - obtener-label: + build-gitian: runs-on: ubuntu-latest steps: From f86bde6980124a0f8276de1c84b74e775df1421c Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Mon, 12 Feb 2024 16:08:00 -0700 Subject: [PATCH 09/46] Update CI.yaml --- .github/workflows/CI.yaml | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 08f000d..a1ed57c 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -32,14 +32,14 @@ jobs: gcloud compute os-login ssh-keys remove --key $i || true; done - gcloud compute instances create test-gitian-$random --image-family=debian-11 --image-project=debian-cloud --machine-type=c2-standard-16 --project=ecc-infra-prod --zone=us-central1-a --no-address --network=vpc-ecc-infra-prod --subnet=us-central1-zcash --tags=zcash --service-account=vm-iap@ecc-infra-prod.iam.gserviceaccount.com --metadata=enable-oslogin=TRUE --scopes=cloud-platform --enable-nested-virtualization --boot-disk-size=200GB + gcloud compute instances create test-gitian-$random --image-family=debian-11 --image-project=debian-cloud --machine-type=c2-standard-16 --project=${{ secrets.GCP_PROJECT_ID_PROD }} --zone=us-central1-a --no-address --network=vpc-${{ secrets.GCP_PROJECT_ID_PROD }} --subnet=us-central1-zcash --tags=zcash --service-account=vm-iap@${{ secrets.GCP_PROJECT_ID_PROD }}.iam.gserviceaccount.com --metadata=enable-oslogin=TRUE --scopes=cloud-platform --enable-nested-virtualization --boot-disk-size=200GB export counter=1 - while [[ $(gcloud compute ssh --zone "us-central1-a" "test-gitian-$random" --tunnel-through-iap --project "ecc-infra-prod" --command="ls -la" &>/dev/null || echo "re-try") == "re-try" && counter -lt 60 ]] + while [[ $(gcloud compute ssh --zone "us-central1-a" "test-gitian-$random" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --command="ls -la" &>/dev/null || echo "re-try") == "re-try" && counter -lt 60 ]] do echo "attemp number: $counter" export counter=$((counter+1)) - if [ $counter -eq 60 ]; then gcloud compute instances delete "test-gitian-$random" --project "ecc-infra-prod" --zone "us-central1-a" --delete-disks=all; exit 1; fi + if [ $counter -eq 60 ]; then gcloud compute instances delete "test-gitian-$random" --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --zone "us-central1-a" --delete-disks=all; exit 1; fi sleep 5 done @@ -102,9 +102,9 @@ jobs: done # get keys - gsutil rm -r gs://ecc-infra-prod-apt-packages/127.0.0.1 || echo "" - gsutil cp gs://ecc-infra-prod-apt-packages/encrypted_gpg.kms \$HOME/encrypted_gpg.kms - gsutil cp gs://ecc-infra-prod-apt-packages/public.asc \$HOME/public.asc + gsutil rm -r gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/127.0.0.1 || echo "" + gsutil cp gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/encrypted_gpg.kms \$HOME/encrypted_gpg.kms + gsutil cp gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/public.asc \$HOME/public.asc current_dir=\$(pwd) cd \$HOME gcloud kms decrypt \ @@ -151,14 +151,14 @@ jobs: cd \$current_dir done export final_version=\$(cat assert2.txt | awk '{print \$2}' | grep "desc.yml" | head -n 1 | sed 's/-desc.yml//g') - gsutil -m rsync -r ./debs gs://ecc-infra-prod-apt-packages/debs - gsutil -m rsync -r ./zcash-binaries gs://ecc-infra-prod-apt-packages/zcash-binaries + gsutil -m rsync -r ./debs gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/debs + gsutil -m rsync -r ./zcash-binaries gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/zcash-binaries apt install aptly -y # generate apt mkdir aptserver cd aptserver - gsutil -m cp -r gs://ecc-infra-prod-apt-server/pool/main/z/zcash/ . + gsutil -m cp -r gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-server/pool/main/z/zcash/ . cd zcash cp -a ../../debs/buster/zcbuild/*.deb \$final_version-amd64-buster.deb cp -a ../../debs/bullseye/zcbuild/*.deb \$final_version-amd64-bullseye.deb @@ -216,39 +216,41 @@ jobs: cp \$HOME/public.asc \$HOME/mirror/127.0.0.1/zcash.asc cd \$HOME/mirror - gsutil -m rsync -r ./127.0.0.1 gs://ecc-infra-prod-apt-packages/127.0.0.1 + gsutil -m rsync -r ./127.0.0.1 gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/127.0.0.1 cd 127.0.0.1 if ! [[ ${array[2]} == *"-rc"* ]]; then - gsutil -m rsync -r ./ gs://ecc-infra-prod-apt-server/ + gsutil -m rsync -r ./ gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-server/ fi EOF export FAIL=0 chmod +x ./script.sh || echo "" - gcloud compute scp ./script.sh --zone "us-central1-a" --tunnel-through-iap --project "ecc-infra-prod" test-gitian-$random: || export FAIL=1 - gcloud compute scp --recurse $(pwd) --zone "us-central1-a" --tunnel-through-iap --project "ecc-infra-prod" test-gitian-$random:~/source || export FAIL=1 + gcloud compute scp ./script.sh --zone "us-central1-a" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random: || export FAIL=1 + gcloud compute scp --recurse $(pwd) --zone "us-central1-a" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random:~/source || export FAIL=1 - gcloud compute ssh --zone "us-central1-a" "test-gitian-$random" --tunnel-through-iap --project "ecc-infra-prod" --command="bash -i -c 'sudo -s ./script.sh'" -- -t || export FAIL=1 + gcloud compute ssh --zone "us-central1-a" "test-gitian-$random" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --command="bash -i -c 'sudo -s ./script.sh'" -- -t || export FAIL=1 - gcloud compute scp --recurse --zone "us-central1-a" --tunnel-through-iap --project "ecc-infra-prod" test-gitian-$random:/home/sa_*/source/gitian.sigs . + gcloud compute scp --recurse --zone "us-central1-a" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random:/home/sa_*/source/gitian.sigs . - gcloud compute instances delete "test-gitian-$random" --project "ecc-infra-prod" --zone "us-central1-a" --delete-disks=all + gcloud compute instances delete "test-gitian-$random" --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --zone "us-central1-a" --delete-disks=all if [ $FAIL -eq 1 ]; then exit 1; fi rm -rf gitian.sigs/.git if ! [[ ${array[2]} == *"-rc"* ]]; then + echo ${{ secrets.BOT_SSH_KEY }} > .ssh/id_rsa + chmod 600 .ssh/id_rsa git clone git@github.com:zcash/gitian.sigs.git sigs cp -a gitian.sigs/* sigs/ cd sigs git config --global user.name "ECC-CI" - git config --global user.email "serviceusers+github@z.cash" + git config --global user.email "${{ secrets.BOT_EMAIL }}" git add . git commit -am "$(inputs.params.LABEL_NAME)" git push fi - curl --request POST --url https://api.bunny.net/pullzone/1432616/purgeCache --header 'content-type: application/json' --header 'AccessKey: ${{ secrets.BUNNY_API_KEY }}' + curl --request POST --url https://api.bunny.net/pullzone/${{ secrets.BUNNY_RESOURCE }}/purgeCache --header 'content-type: application/json' --header 'AccessKey: ${{ secrets.BUNNY_API_KEY }}' shell: bash From 3b63dc8e9989020797a909198e185ba438beb570 Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Mon, 12 Feb 2024 19:34:38 -0700 Subject: [PATCH 10/46] Update CI.yaml --- .github/workflows/CI.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index a1ed57c..3c55aaf 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -235,12 +235,14 @@ jobs: gcloud compute instances delete "test-gitian-$random" --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --zone "us-central1-a" --delete-disks=all + curl --request POST --url https://api.bunny.net/pullzone/${{ secrets.BUNNY_RESOURCE }}/purgeCache --header 'content-type: application/json' --header 'AccessKey: ${{ secrets.BUNNY_API_KEY }}' if [ $FAIL -eq 1 ]; then exit 1; fi rm -rf gitian.sigs/.git if ! [[ ${array[2]} == *"-rc"* ]]; then - echo ${{ secrets.BOT_SSH_KEY }} > .ssh/id_rsa - chmod 600 .ssh/id_rsa + mkdir $HOME/.ssh || echo "" + echo "${{ secrets.BOT_SSH_KEY }}" > $HOME/.ssh/id_rsa + chmod 600 $HOME/.ssh/id_rsa git clone git@github.com:zcash/gitian.sigs.git sigs cp -a gitian.sigs/* sigs/ cd sigs @@ -251,6 +253,6 @@ jobs: git push fi - curl --request POST --url https://api.bunny.net/pullzone/${{ secrets.BUNNY_RESOURCE }}/purgeCache --header 'content-type: application/json' --header 'AccessKey: ${{ secrets.BUNNY_API_KEY }}' + shell: bash From cd075600e3b4f62181bc5e42ca5e5c7336b8106b Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Tue, 13 Feb 2024 06:39:49 -0700 Subject: [PATCH 11/46] Update .github/workflows/CI.yaml Co-authored-by: Daira-Emma Hopwood --- .github/workflows/CI.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 3c55aaf..d25daf2 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -25,7 +25,7 @@ jobs: echo ${{ secrets.GCP_SA_KEY }} | base64 -d > json.json gcloud auth activate-service-account --key-file=json.json - export random=$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 4; echo;) + export random=$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 12; echo;) for i in $(gcloud compute os-login ssh-keys list --format="table[no-heading](value.fingerprint)"); do echo $i; From c0acb86167cb1d565b5437bf6d9881afe6f59d5d Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Tue, 13 Feb 2024 06:40:11 -0700 Subject: [PATCH 12/46] Update .github/workflows/CI.yaml Co-authored-by: Daira-Emma Hopwood --- .github/workflows/CI.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index d25daf2..1b7f7da 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -37,7 +37,7 @@ jobs: export counter=1 while [[ $(gcloud compute ssh --zone "us-central1-a" "test-gitian-$random" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --command="ls -la" &>/dev/null || echo "re-try") == "re-try" && counter -lt 60 ]] do - echo "attemp number: $counter" + echo "attempt number: $counter" export counter=$((counter+1)) if [ $counter -eq 60 ]; then gcloud compute instances delete "test-gitian-$random" --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --zone "us-central1-a" --delete-disks=all; exit 1; fi sleep 5 From 659a1ae2952bb83d3c50694fdbada47a58d30d9b Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Tue, 13 Feb 2024 06:40:26 -0700 Subject: [PATCH 13/46] Update .github/workflows/CI.yaml Co-authored-by: Daira-Emma Hopwood --- .github/workflows/CI.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 1b7f7da..3333bac 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -28,7 +28,7 @@ jobs: export random=$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 12; echo;) for i in $(gcloud compute os-login ssh-keys list --format="table[no-heading](value.fingerprint)"); do - echo $i; + echo "removing SSH key $i" gcloud compute os-login ssh-keys remove --key $i || true; done From 19aed3b3236a10d6fb0ed755ec4c89765f5f154d Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Tue, 13 Feb 2024 06:40:45 -0700 Subject: [PATCH 14/46] Update .github/workflows/CI.yaml Co-authored-by: Daira-Emma Hopwood --- .github/workflows/CI.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 3333bac..756158f 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -29,7 +29,7 @@ jobs: for i in $(gcloud compute os-login ssh-keys list --format="table[no-heading](value.fingerprint)"); do echo "removing SSH key $i" - gcloud compute os-login ssh-keys remove --key $i || true; + gcloud compute os-login ssh-keys remove --key $i || echo "failed to remove key" done gcloud compute instances create test-gitian-$random --image-family=debian-11 --image-project=debian-cloud --machine-type=c2-standard-16 --project=${{ secrets.GCP_PROJECT_ID_PROD }} --zone=us-central1-a --no-address --network=vpc-${{ secrets.GCP_PROJECT_ID_PROD }} --subnet=us-central1-zcash --tags=zcash --service-account=vm-iap@${{ secrets.GCP_PROJECT_ID_PROD }}.iam.gserviceaccount.com --metadata=enable-oslogin=TRUE --scopes=cloud-platform --enable-nested-virtualization --boot-disk-size=200GB From 8e6155b3d3ec2f08dc344d54c53bc39ffddaa512 Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Tue, 13 Feb 2024 06:46:14 -0700 Subject: [PATCH 15/46] check sha256sum of yq binary --- .github/workflows/CI.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 756158f..8c408dc 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -48,6 +48,7 @@ jobs: git clone -b ${array[2]} https://github.com/${array[0]}/${array[1]}.git cd zcash/contrib/gitian-descriptors wget -c https://github.com/mikefarah/yq/releases/download/v4.28.2/yq_linux_amd64 + echo "7e0d59c65be5054a14ff2a76eb12c2d4ec3e5bc2f1dfa03c7356bb35b50bbf41 yq_linux_amd64" | shasum -a 256 -c chmod +x yq_linux_amd64 export ZCASH_GITIAN_VERSION=$(cat gitian-linux-parallel.yml | ./yq_linux_amd64 .name) cd ../../.. From f61329ef0e64c327b5dc673bd391c779b3239dbc Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Tue, 13 Feb 2024 07:00:33 -0700 Subject: [PATCH 16/46] pin hashes --- .github/workflows/CI.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 8c408dc..780b52e 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -58,12 +58,16 @@ jobs: apt update; apt install ca-certificates curl gnupg lsb-release zsh software-properties-common wget git vagrant python3-venv direnv python3-pip linux-headers-\$(uname -r) ansible -y; mkdir -m 0755 -p /etc/apt/keyrings; - curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg --yes; + curl -fsSL https://download.docker.com/linux/debian/gpg -o gpg + echo "1500c1f56fa9e26b9b8f42452a553675796ade0807cdce11975eb98170b3a570 gpg" | shasum -a 256 -c; + sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg --yes < gpg; echo "deb [arch=\$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \$(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null apt update; apt install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y; apt-add-repository "deb http://download.virtualbox.org/virtualbox/debian \$(lsb_release -sc) contrib"; - wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc -O- | apt-key add -; + wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc; + echo "49e6801d45f6536232c11be6cdb43fa8e0198538d29d1075a7e10165e1fbafe2 oracle_vbox_2016.asc" | shasum -a 256 -c; + apt-key add oracle_vbox_2016.asc; apt update apt install virtualbox-6.1 -y; eval "\$(direnv hook bash)"; From 229ec11a59b8f377a8199a8debb6e24650d9078f Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Tue, 13 Feb 2024 07:00:58 -0700 Subject: [PATCH 17/46] Change author of gpg Co-authored-by: Daira-Emma Hopwood --- .github/workflows/CI.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 780b52e..fccfe17 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -85,7 +85,7 @@ jobs: /sbin/vboxconfig; vagrant plugin install --local; vagrant plugin install --local; - gpg --quick-generate-key --batch --passphrase '' "Harry Potter (zcash gitian) " + gpg --quick-generate-key --batch --passphrase '' "Lyra Silvertongue (zcash gitian) " echo "GPG_KEY_ID=\$(gpg --list-keys --with-fingerprint --with-colons | grep fpr: | head -n 1 | sed 's/fpr://g' | sed 's/://g')" >> .env; git config --global user.name "Harry Potter" git config --global user.email "hpotter@hogwarts.wiz" From b86c3878094e255cc385498709b018e41b9fdef2 Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Tue, 13 Feb 2024 07:01:17 -0700 Subject: [PATCH 18/46] change author of git Co-authored-by: Daira-Emma Hopwood --- .github/workflows/CI.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index fccfe17..ac1829c 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -87,7 +87,7 @@ jobs: vagrant plugin install --local; gpg --quick-generate-key --batch --passphrase '' "Lyra Silvertongue (zcash gitian) " echo "GPG_KEY_ID=\$(gpg --list-keys --with-fingerprint --with-colons | grep fpr: | head -n 1 | sed 's/fpr://g' | sed 's/://g')" >> .env; - git config --global user.name "Harry Potter" + git config --global user.name "Lyra Silvertongue" git config --global user.email "hpotter@hogwarts.wiz" direnv allow; direnv exec \$(pwd) vagrant up zcash-build; From 3131d37a61f0665d813d0cb6fd1e4f6cf0e57064 Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Tue, 13 Feb 2024 07:01:34 -0700 Subject: [PATCH 19/46] change email of author in git Co-authored-by: Daira-Emma Hopwood --- .github/workflows/CI.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index ac1829c..779f377 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -88,7 +88,7 @@ jobs: gpg --quick-generate-key --batch --passphrase '' "Lyra Silvertongue (zcash gitian) " echo "GPG_KEY_ID=\$(gpg --list-keys --with-fingerprint --with-colons | grep fpr: | head -n 1 | sed 's/fpr://g' | sed 's/://g')" >> .env; git config --global user.name "Lyra Silvertongue" - git config --global user.email "hpotter@hogwarts.wiz" + git config --global user.email "lyra.silvertongue@ox.ac.brytain" direnv allow; direnv exec \$(pwd) vagrant up zcash-build; vagrant ssh zcash-build -c "gpg --quick-generate-key --batch --passphrase '' \"Harry Potter (zcash gitian) \" || echo ''" From 6b8bab85494243a2fb364e9c20f31a31b490b573 Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Tue, 13 Feb 2024 07:01:48 -0700 Subject: [PATCH 20/46] author gpg Co-authored-by: Daira-Emma Hopwood --- .github/workflows/CI.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 779f377..2d3757a 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -91,7 +91,7 @@ jobs: git config --global user.email "lyra.silvertongue@ox.ac.brytain" direnv allow; direnv exec \$(pwd) vagrant up zcash-build; - vagrant ssh zcash-build -c "gpg --quick-generate-key --batch --passphrase '' \"Harry Potter (zcash gitian) \" || echo ''" + vagrant ssh zcash-build -c "gpg --quick-generate-key --batch --passphrase '' \"Lyra Silvertongue (zcash gitian) \" || echo ''" vagrant ssh zcash-build -c ./gitian-parallel-build.sh || exit 1 vagrant ssh zcash-build -c "head -n 8 gitian.sigs/\$VERSION*/hpotter/*.assert" > assert.txt tr -d \$'\r' < assert.txt > assert2.txt From 17ccc42a3e92d9052aba06ef64c23a6ada2ddd5e Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Tue, 13 Feb 2024 07:02:07 -0700 Subject: [PATCH 21/46] author of git Co-authored-by: Daira-Emma Hopwood --- .github/workflows/CI.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 2d3757a..4063614 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -93,7 +93,7 @@ jobs: direnv exec \$(pwd) vagrant up zcash-build; vagrant ssh zcash-build -c "gpg --quick-generate-key --batch --passphrase '' \"Lyra Silvertongue (zcash gitian) \" || echo ''" vagrant ssh zcash-build -c ./gitian-parallel-build.sh || exit 1 - vagrant ssh zcash-build -c "head -n 8 gitian.sigs/\$VERSION*/hpotter/*.assert" > assert.txt + vagrant ssh zcash-build -c "head -n 8 gitian.sigs/\$VERSION*/lyra.silvertongue/*.assert" > assert.txt tr -d \$'\r' < assert.txt > assert2.txt for i in \$(cat assert2.txt | grep -E "zcash-*" | grep -v git: | sed 's/ //g' | sed 's/ /-->/g'); do curl ${{ secrets.SLACK_WEBHOOK }} -H "Content-Type: application/json" -d "{\"text\": \"\\\`\\\`\\\`\$i\\\`\\\`\\\`\"}" From c8a02fd8885470f53172c24d1fbf9b4fe7c791a7 Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Tue, 13 Feb 2024 07:09:11 -0700 Subject: [PATCH 22/46] delete upgrade of pip Co-authored-by: Daira-Emma Hopwood --- .github/workflows/CI.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 4063614..2ff2b67 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -81,7 +81,6 @@ jobs: echo "ZCASH_GIT_REPO_URL=https://github.com/${array[0]}/${array[1]}" >> .env; cat .env direnv allow; - pip3 install --upgrade pip; /sbin/vboxconfig; vagrant plugin install --local; vagrant plugin install --local; From 9e913ce457d3da15a33c125a9b6e09ef0fb90b5b Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Tue, 13 Feb 2024 14:03:56 -0700 Subject: [PATCH 23/46] Update CI.yaml --- .github/workflows/CI.yaml | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 2ff2b67..619bcad 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -20,7 +20,24 @@ jobs: - name: Build Gitian id: gitian + shell: bash run: | + ############################################################################################## + set -eE -o functrace + + failure() { + local lineno=$2 + local fn=$3 + local exitstatus=$4 + local msg=$5 + local lineno_fns=${1% 0} + if [[ "$lineno_fns" != "0" ]] ; then + lineno="${lineno} ${lineno_fns}" + fi + echo "${BASH_SOURCE[1]}:${fn}[${lineno}] Failed with status ${exitstatus}: $msg" + } + trap 'failure "${BASH_LINENO[*]}" "$LINENO" "${FUNCNAME[*]:-script}" "$?" "$BASH_COMMAND"' ERR + ############################################################################################## sudo apt update; sudo apt install wget openssh-client git -y echo ${{ secrets.GCP_SA_KEY }} | base64 -d > json.json gcloud auth activate-service-account --key-file=json.json @@ -55,8 +72,25 @@ jobs: cat < ./script.sh + #!/bin/bash + ############################################################################################## + set -eE -o functrace + + failure() { + local lineno=$2 + local fn=$3 + local exitstatus=$4 + local msg=$5 + local lineno_fns=${1% 0} + if [[ "$lineno_fns" != "0" ]] ; then + lineno="${lineno} ${lineno_fns}" + fi + echo "${BASH_SOURCE[1]}:${fn}[${lineno}] Failed with status ${exitstatus}: $msg" + } + trap 'failure "${BASH_LINENO[*]}" "$LINENO" "${FUNCNAME[*]:-script}" "$?" "$BASH_COMMAND"' ERR + ############################################################################################## apt update; - apt install ca-certificates curl gnupg lsb-release zsh software-properties-common wget git vagrant python3-venv direnv python3-pip linux-headers-\$(uname -r) ansible -y; + apt install ca-certificates curl gnupg lsb-release software-properties-common wget git vagrant python3-venv direnv python3-pip linux-headers-\$(uname -r) ansible -y; mkdir -m 0755 -p /etc/apt/keyrings; curl -fsSL https://download.docker.com/linux/debian/gpg -o gpg echo "1500c1f56fa9e26b9b8f42452a553675796ade0807cdce11975eb98170b3a570 gpg" | shasum -a 256 -c; From 7d731ec9d209fabff576789c979efc1d0fdf93e7 Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Tue, 13 Feb 2024 14:30:36 -0700 Subject: [PATCH 24/46] Update CI.yaml --- .github/workflows/CI.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 619bcad..38a0863 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -290,7 +290,3 @@ jobs: git commit -am "$(inputs.params.LABEL_NAME)" git push fi - - - - shell: bash From d6facd33638c40daf888f7cf9ac0495e5eefbb41 Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Tue, 13 Feb 2024 15:02:32 -0700 Subject: [PATCH 25/46] Update CI.yaml --- .github/workflows/CI.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 38a0863..bb3ab09 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -7,7 +7,7 @@ on: jobs: build-gitian: - runs-on: ubuntu-latest + runs-on: docker # ubuntu-latest steps: - name: Checkout code From fe30f629a23c750aad75302f3aece28273b1aabb Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Tue, 13 Feb 2024 17:14:21 -0700 Subject: [PATCH 26/46] Update CI.yaml --- .github/workflows/CI.yaml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index bb3ab09..5a19349 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -7,7 +7,7 @@ on: jobs: build-gitian: - runs-on: docker # ubuntu-latest + runs-on: ubuntu-latest steps: - name: Checkout code @@ -64,7 +64,7 @@ jobs: git clone -b ${array[2]} https://github.com/${array[0]}/${array[1]}.git cd zcash/contrib/gitian-descriptors - wget -c https://github.com/mikefarah/yq/releases/download/v4.28.2/yq_linux_amd64 + wget -c -q https://github.com/mikefarah/yq/releases/download/v4.28.2/yq_linux_amd64 echo "7e0d59c65be5054a14ff2a76eb12c2d4ec3e5bc2f1dfa03c7356bb35b50bbf41 yq_linux_amd64" | shasum -a 256 -c chmod +x yq_linux_amd64 export ZCASH_GITIAN_VERSION=$(cat gitian-linux-parallel.yml | ./yq_linux_amd64 .name) @@ -129,20 +129,20 @@ jobs: vagrant ssh zcash-build -c "head -n 8 gitian.sigs/\$VERSION*/lyra.silvertongue/*.assert" > assert.txt tr -d \$'\r' < assert.txt > assert2.txt for i in \$(cat assert2.txt | grep -E "zcash-*" | grep -v git: | sed 's/ //g' | sed 's/ /-->/g'); do - curl ${{ secrets.SLACK_WEBHOOK }} -H "Content-Type: application/json" -d "{\"text\": \"\\\`\\\`\\\`\$i\\\`\\\`\\\`\"}" + curl -s ${{ secrets.SLACK_WEBHOOK }} -H "Content-Type: application/json" -d "{\"text\": \"\\\`\\\`\\\`\$i\\\`\\\`\\\`\"}" done export OS=\$(vagrant ssh zcash-build -c "ls zcash-binaries/\$VERSION" | tr -d '\r') for i in \$OS; do vagrant ssh zcash-build -c "mkdir \$i; tar Cxvzf \$i zcash-binaries/*/\$i/zcash-*-linux64.tar.gz"; done versions=\$(for i in \$OS; do echo " \$i==>"; vagrant ssh zcash-build -c "./\$i/zcash-*/bin/zcashd --version | head -n 1 | tr -d '\n'"; done) for i in "\${versions[@]}" do - curl ${{ secrets.SLACK_WEBHOOK }} -H "Content-Type: application/json" -d "{\"text\": \"\\\`\\\`\\\`\$i\\\`\\\`\\\`\"}" + curl -s ${{ secrets.SLACK_WEBHOOK }} -H "Content-Type: application/json" -d "{\"text\": \"\\\`\\\`\\\`\$i\\\`\\\`\\\`\"}" done # get keys - gsutil rm -r gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/127.0.0.1 || echo "" - gsutil cp gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/encrypted_gpg.kms \$HOME/encrypted_gpg.kms - gsutil cp gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/public.asc \$HOME/public.asc + gsutil -q rm -r gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/127.0.0.1 || echo "" + gsutil -q cp gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/encrypted_gpg.kms \$HOME/encrypted_gpg.kms + gsutil -q cp gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/public.asc \$HOME/public.asc current_dir=\$(pwd) cd \$HOME gcloud kms decrypt \ @@ -189,14 +189,14 @@ jobs: cd \$current_dir done export final_version=\$(cat assert2.txt | awk '{print \$2}' | grep "desc.yml" | head -n 1 | sed 's/-desc.yml//g') - gsutil -m rsync -r ./debs gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/debs - gsutil -m rsync -r ./zcash-binaries gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/zcash-binaries + gsutil -q -m rsync -r ./debs gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/debs + gsutil -q -m rsync -r ./zcash-binaries gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/zcash-binaries apt install aptly -y # generate apt mkdir aptserver cd aptserver - gsutil -m cp -r gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-server/pool/main/z/zcash/ . + gsutil -q -m cp -r gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-server/pool/main/z/zcash/ . cd zcash cp -a ../../debs/buster/zcbuild/*.deb \$final_version-amd64-buster.deb cp -a ../../debs/bullseye/zcbuild/*.deb \$final_version-amd64-bullseye.deb @@ -250,14 +250,14 @@ jobs: /etc/init.d/nginx restart mkdir \$HOME/mirror cd \$HOME/mirror - wget -r 127.0.0.1 + wget -q -r 127.0.0.1 cp \$HOME/public.asc \$HOME/mirror/127.0.0.1/zcash.asc cd \$HOME/mirror - gsutil -m rsync -r ./127.0.0.1 gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/127.0.0.1 + gsutil -q -m rsync -r ./127.0.0.1 gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/127.0.0.1 cd 127.0.0.1 if ! [[ ${array[2]} == *"-rc"* ]]; then - gsutil -m rsync -r ./ gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-server/ + gsutil -q -m rsync -r ./ gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-server/ fi EOF @@ -273,7 +273,7 @@ jobs: gcloud compute instances delete "test-gitian-$random" --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --zone "us-central1-a" --delete-disks=all - curl --request POST --url https://api.bunny.net/pullzone/${{ secrets.BUNNY_RESOURCE }}/purgeCache --header 'content-type: application/json' --header 'AccessKey: ${{ secrets.BUNNY_API_KEY }}' + curl -s --request POST --url https://api.bunny.net/pullzone/${{ secrets.BUNNY_RESOURCE }}/purgeCache --header 'content-type: application/json' --header 'AccessKey: ${{ secrets.BUNNY_API_KEY }}' if [ $FAIL -eq 1 ]; then exit 1; fi rm -rf gitian.sigs/.git From 48f8d5d86c2c79489ffe81f6901fee24ab4bc20b Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Tue, 13 Feb 2024 17:25:18 -0700 Subject: [PATCH 27/46] Update CI.yaml --- .github/workflows/CI.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 5a19349..23d3d31 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -271,7 +271,7 @@ jobs: gcloud compute scp --recurse --zone "us-central1-a" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random:/home/sa_*/source/gitian.sigs . - gcloud compute instances delete "test-gitian-$random" --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --zone "us-central1-a" --delete-disks=all + #gcloud compute instances delete "test-gitian-$random" --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --zone "us-central1-a" --delete-disks=all curl -s --request POST --url https://api.bunny.net/pullzone/${{ secrets.BUNNY_RESOURCE }}/purgeCache --header 'content-type: application/json' --header 'AccessKey: ${{ secrets.BUNNY_API_KEY }}' if [ $FAIL -eq 1 ]; then exit 1; fi From cd88f070f66024e45c15a5a2a826ce11a05d829f Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Tue, 13 Feb 2024 17:34:39 -0700 Subject: [PATCH 28/46] Update CI.yaml --- .github/workflows/CI.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 23d3d31..86bb234 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -77,17 +77,17 @@ jobs: set -eE -o functrace failure() { - local lineno=$2 - local fn=$3 - local exitstatus=$4 - local msg=$5 - local lineno_fns=${1% 0} - if [[ "$lineno_fns" != "0" ]] ; then - lineno="${lineno} ${lineno_fns}" + local lineno=\$2 + local fn=\$3 + local exitstatus=\$4 + local msg=\$5 + local lineno_fns=\${1% 0} + if [[ "\$lineno_fns" != "0" ]] ; then + lineno="\${lineno} \${lineno_fns}" fi - echo "${BASH_SOURCE[1]}:${fn}[${lineno}] Failed with status ${exitstatus}: $msg" + echo "\${BASH_SOURCE[1]}:\${fn}[\${lineno}] Failed with status \${exitstatus}: \$msg" } - trap 'failure "${BASH_LINENO[*]}" "$LINENO" "${FUNCNAME[*]:-script}" "$?" "$BASH_COMMAND"' ERR + trap 'failure "\${BASH_LINENO[*]}" "\$LINENO" "\${FUNCNAME[*]:-script}" "\$?" "\$BASH_COMMAND"' ERR ############################################################################################## apt update; apt install ca-certificates curl gnupg lsb-release software-properties-common wget git vagrant python3-venv direnv python3-pip linux-headers-\$(uname -r) ansible -y; From 2aa3b9edb5f852bf6b68e25004ee2be40be8768d Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Tue, 13 Feb 2024 17:47:09 -0700 Subject: [PATCH 29/46] fix name on docker gpg Co-authored-by: Daira-Emma Hopwood --- .github/workflows/CI.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 86bb234..42bd4db 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -92,9 +92,9 @@ jobs: apt update; apt install ca-certificates curl gnupg lsb-release software-properties-common wget git vagrant python3-venv direnv python3-pip linux-headers-\$(uname -r) ansible -y; mkdir -m 0755 -p /etc/apt/keyrings; - curl -fsSL https://download.docker.com/linux/debian/gpg -o gpg - echo "1500c1f56fa9e26b9b8f42452a553675796ade0807cdce11975eb98170b3a570 gpg" | shasum -a 256 -c; - sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg --yes < gpg; + curl -fsSL https://download.docker.com/linux/debian/gpg -o gpg.asc + echo "1500c1f56fa9e26b9b8f42452a553675796ade0807cdce11975eb98170b3a570 gpg.asc" | shasum -a 256 -c; + sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg --yes < gpg.asc; echo "deb [arch=\$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \$(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null apt update; apt install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y; From 039fdfee992bd85c7e7ad6952e297f7e4992c238 Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Tue, 13 Feb 2024 17:57:25 -0700 Subject: [PATCH 30/46] silent apt --- .github/workflows/CI.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 42bd4db..19d71a7 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -38,7 +38,7 @@ jobs: } trap 'failure "${BASH_LINENO[*]}" "$LINENO" "${FUNCNAME[*]:-script}" "$?" "$BASH_COMMAND"' ERR ############################################################################################## - sudo apt update; sudo apt install wget openssh-client git -y + sudo apt -qq update; sudo apt install wget openssh-client git -y >/dev/null echo ${{ secrets.GCP_SA_KEY }} | base64 -d > json.json gcloud auth activate-service-account --key-file=json.json @@ -89,21 +89,21 @@ jobs: } trap 'failure "\${BASH_LINENO[*]}" "\$LINENO" "\${FUNCNAME[*]:-script}" "\$?" "\$BASH_COMMAND"' ERR ############################################################################################## - apt update; - apt install ca-certificates curl gnupg lsb-release software-properties-common wget git vagrant python3-venv direnv python3-pip linux-headers-\$(uname -r) ansible -y; + apt -qq update; + apt install ca-certificates curl gnupg lsb-release software-properties-common wget git vagrant python3-venv direnv python3-pip linux-headers-\$(uname -r) ansible -y >/dev/null; mkdir -m 0755 -p /etc/apt/keyrings; curl -fsSL https://download.docker.com/linux/debian/gpg -o gpg.asc echo "1500c1f56fa9e26b9b8f42452a553675796ade0807cdce11975eb98170b3a570 gpg.asc" | shasum -a 256 -c; sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg --yes < gpg.asc; echo "deb [arch=\$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \$(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - apt update; - apt install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y; + apt -qq update; + apt install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y >/dev/null; apt-add-repository "deb http://download.virtualbox.org/virtualbox/debian \$(lsb_release -sc) contrib"; wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc; echo "49e6801d45f6536232c11be6cdb43fa8e0198538d29d1075a7e10165e1fbafe2 oracle_vbox_2016.asc" | shasum -a 256 -c; apt-key add oracle_vbox_2016.asc; - apt update - apt install virtualbox-6.1 -y; + apt -qq update + apt install virtualbox-6.1 -y >/dev/null; eval "\$(direnv hook bash)"; cd source cp .env.example .env @@ -163,7 +163,7 @@ jobs: docker run -d --name \$i debian:\$i bash -c "while true; do sleep 2; done"; - docker exec \$i bash -c "mkdir -p /home/vagrant/\$i-deb-build && cd /home/vagrant/\$i-deb-build && apt update && apt install git dpkg-dev lintian -y && git clone -b ${array[2]} https://github.com/${array[0]}/${array[1]}.git ."; + docker exec \$i bash -c "mkdir -p /home/vagrant/\$i-deb-build && cd /home/vagrant/\$i-deb-build && apt -qq update && apt install git dpkg-dev lintian -y && git clone -b ${array[2]} https://github.com/${array[0]}/${array[1]}.git ."; vagrant scp :/home/vagrant/\$i-extract/zcash-*/bin/zcash-tx ./\$i-extract/ vagrant scp :/home/vagrant/\$i-extract/zcash-*/bin/zcash-fetch-params ./\$i-extract/ @@ -191,7 +191,7 @@ jobs: export final_version=\$(cat assert2.txt | awk '{print \$2}' | grep "desc.yml" | head -n 1 | sed 's/-desc.yml//g') gsutil -q -m rsync -r ./debs gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/debs gsutil -q -m rsync -r ./zcash-binaries gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/zcash-binaries - apt install aptly -y + apt install aptly -y >/dev/null # generate apt mkdir aptserver @@ -233,7 +233,7 @@ jobs: aptly publish snapshot --distribution bullseye --component main --architectures amd64 --gpg-key="\$key" --passphrase="" bullseye_snapshot aptly publish snapshot --distribution stretch --component main --architectures amd64 --gpg-key="\$key" --passphrase="" stretch_snapshot - apt install nginx-extras -y + apt install nginx-extras -y >/dev/null cat << EOH > /etc/nginx/sites-enabled/default server { listen 80 default_server; From 7bb892b87e0d4de78c340f3177815c9ba17852b2 Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Tue, 13 Feb 2024 19:07:30 -0700 Subject: [PATCH 31/46] FIX GPG for Lyra Silvertongue --- .github/workflows/CI.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 19d71a7..7e4f69f 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -120,6 +120,7 @@ jobs: vagrant plugin install --local; gpg --quick-generate-key --batch --passphrase '' "Lyra Silvertongue (zcash gitian) " echo "GPG_KEY_ID=\$(gpg --list-keys --with-fingerprint --with-colons | grep fpr: | head -n 1 | sed 's/fpr://g' | sed 's/://g')" >> .env; + echo "GPG_KEY_NAME=lyra.silvertongue" >> .env; git config --global user.name "Lyra Silvertongue" git config --global user.email "lyra.silvertongue@ox.ac.brytain" direnv allow; From b5e47eaa607b656817feef312a32d72096099015 Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Sat, 17 Feb 2024 08:40:00 -0700 Subject: [PATCH 32/46] Update CI.yaml --- .github/workflows/CI.yaml | 110 ++------------------------------------ 1 file changed, 3 insertions(+), 107 deletions(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 7e4f69f..aba1a73 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -1,77 +1,3 @@ -name: Gitian CI - -on: - pull_request: - types: - - labeled - -jobs: - build-gitian: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v2 - - - name: 'Set up Cloud SDK' - uses: 'google-github-actions/setup-gcloud@v2' - with: - version: '>= 363.0.0' - - - name: Build Gitian - id: gitian - shell: bash - run: | - ############################################################################################## - set -eE -o functrace - - failure() { - local lineno=$2 - local fn=$3 - local exitstatus=$4 - local msg=$5 - local lineno_fns=${1% 0} - if [[ "$lineno_fns" != "0" ]] ; then - lineno="${lineno} ${lineno_fns}" - fi - echo "${BASH_SOURCE[1]}:${fn}[${lineno}] Failed with status ${exitstatus}: $msg" - } - trap 'failure "${BASH_LINENO[*]}" "$LINENO" "${FUNCNAME[*]:-script}" "$?" "$BASH_COMMAND"' ERR - ############################################################################################## - sudo apt -qq update; sudo apt install wget openssh-client git -y >/dev/null - echo ${{ secrets.GCP_SA_KEY }} | base64 -d > json.json - gcloud auth activate-service-account --key-file=json.json - - export random=$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 12; echo;) - - for i in $(gcloud compute os-login ssh-keys list --format="table[no-heading](value.fingerprint)"); do - echo "removing SSH key $i" - gcloud compute os-login ssh-keys remove --key $i || echo "failed to remove key" - done - - gcloud compute instances create test-gitian-$random --image-family=debian-11 --image-project=debian-cloud --machine-type=c2-standard-16 --project=${{ secrets.GCP_PROJECT_ID_PROD }} --zone=us-central1-a --no-address --network=vpc-${{ secrets.GCP_PROJECT_ID_PROD }} --subnet=us-central1-zcash --tags=zcash --service-account=vm-iap@${{ secrets.GCP_PROJECT_ID_PROD }}.iam.gserviceaccount.com --metadata=enable-oslogin=TRUE --scopes=cloud-platform --enable-nested-virtualization --boot-disk-size=200GB - - export counter=1 - while [[ $(gcloud compute ssh --zone "us-central1-a" "test-gitian-$random" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --command="ls -la" &>/dev/null || echo "re-try") == "re-try" && counter -lt 60 ]] - do - echo "attempt number: $counter" - export counter=$((counter+1)) - if [ $counter -eq 60 ]; then gcloud compute instances delete "test-gitian-$random" --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --zone "us-central1-a" --delete-disks=all; exit 1; fi - sleep 5 - done - - IFS='/' read -r -a array <<< "${{ github.event.label.name }}" - - git clone -b ${array[2]} https://github.com/${array[0]}/${array[1]}.git - cd zcash/contrib/gitian-descriptors - wget -c -q https://github.com/mikefarah/yq/releases/download/v4.28.2/yq_linux_amd64 - echo "7e0d59c65be5054a14ff2a76eb12c2d4ec3e5bc2f1dfa03c7356bb35b50bbf41 yq_linux_amd64" | shasum -a 256 -c - chmod +x yq_linux_amd64 - export ZCASH_GITIAN_VERSION=$(cat gitian-linux-parallel.yml | ./yq_linux_amd64 .name) - cd ../../.. - - - cat < ./script.sh #!/bin/bash ############################################################################################## set -eE -o functrace @@ -129,15 +55,16 @@ jobs: vagrant ssh zcash-build -c ./gitian-parallel-build.sh || exit 1 vagrant ssh zcash-build -c "head -n 8 gitian.sigs/\$VERSION*/lyra.silvertongue/*.assert" > assert.txt tr -d \$'\r' < assert.txt > assert2.txt + echo "#### sigs ####" for i in \$(cat assert2.txt | grep -E "zcash-*" | grep -v git: | sed 's/ //g' | sed 's/ /-->/g'); do - curl -s ${{ secrets.SLACK_WEBHOOK }} -H "Content-Type: application/json" -d "{\"text\": \"\\\`\\\`\\\`\$i\\\`\\\`\\\`\"}" + echo \$i done export OS=\$(vagrant ssh zcash-build -c "ls zcash-binaries/\$VERSION" | tr -d '\r') for i in \$OS; do vagrant ssh zcash-build -c "mkdir \$i; tar Cxvzf \$i zcash-binaries/*/\$i/zcash-*-linux64.tar.gz"; done versions=\$(for i in \$OS; do echo " \$i==>"; vagrant ssh zcash-build -c "./\$i/zcash-*/bin/zcashd --version | head -n 1 | tr -d '\n'"; done) for i in "\${versions[@]}" do - curl -s ${{ secrets.SLACK_WEBHOOK }} -H "Content-Type: application/json" -d "{\"text\": \"\\\`\\\`\\\`\$i\\\`\\\`\\\`\"}" + echo \$i done # get keys @@ -260,34 +187,3 @@ jobs: if ! [[ ${array[2]} == *"-rc"* ]]; then gsutil -q -m rsync -r ./ gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-server/ fi - EOF - - export FAIL=0 - chmod +x ./script.sh || echo "" - - gcloud compute scp ./script.sh --zone "us-central1-a" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random: || export FAIL=1 - gcloud compute scp --recurse $(pwd) --zone "us-central1-a" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random:~/source || export FAIL=1 - - gcloud compute ssh --zone "us-central1-a" "test-gitian-$random" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --command="bash -i -c 'sudo -s ./script.sh'" -- -t || export FAIL=1 - - gcloud compute scp --recurse --zone "us-central1-a" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random:/home/sa_*/source/gitian.sigs . - - #gcloud compute instances delete "test-gitian-$random" --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --zone "us-central1-a" --delete-disks=all - - curl -s --request POST --url https://api.bunny.net/pullzone/${{ secrets.BUNNY_RESOURCE }}/purgeCache --header 'content-type: application/json' --header 'AccessKey: ${{ secrets.BUNNY_API_KEY }}' - if [ $FAIL -eq 1 ]; then exit 1; fi - - rm -rf gitian.sigs/.git - if ! [[ ${array[2]} == *"-rc"* ]]; then - mkdir $HOME/.ssh || echo "" - echo "${{ secrets.BOT_SSH_KEY }}" > $HOME/.ssh/id_rsa - chmod 600 $HOME/.ssh/id_rsa - git clone git@github.com:zcash/gitian.sigs.git sigs - cp -a gitian.sigs/* sigs/ - cd sigs - git config --global user.name "ECC-CI" - git config --global user.email "${{ secrets.BOT_EMAIL }}" - git add . - git commit -am "$(inputs.params.LABEL_NAME)" - git push - fi From 6f1fa2c66ea8c568630f91d4d1ec19c416fd3c71 Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Sat, 17 Feb 2024 08:41:38 -0700 Subject: [PATCH 33/46] Update CI.yaml --- .github/workflows/CI.yaml | 105 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index aba1a73..b606caf 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -1,3 +1,77 @@ +name: Gitian CI + +on: + pull_request: + types: + - labeled + +jobs: + build-gitian: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v2' + with: + version: '>= 363.0.0' + + - name: Build Gitian + id: gitian + shell: bash + run: | + ############################################################################################## + set -eE -o functrace + + failure() { + local lineno=$2 + local fn=$3 + local exitstatus=$4 + local msg=$5 + local lineno_fns=${1% 0} + if [[ "$lineno_fns" != "0" ]] ; then + lineno="${lineno} ${lineno_fns}" + fi + echo "${BASH_SOURCE[1]}:${fn}[${lineno}] Failed with status ${exitstatus}: $msg" + } + trap 'failure "${BASH_LINENO[*]}" "$LINENO" "${FUNCNAME[*]:-script}" "$?" "$BASH_COMMAND"' ERR + ############################################################################################## + sudo apt -qq update; sudo apt install wget openssh-client git -y >/dev/null + echo ${{ secrets.GCP_SA_KEY }} | base64 -d > json.json + gcloud auth activate-service-account --key-file=json.json + + export random=$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 12; echo;) + + for i in $(gcloud compute os-login ssh-keys list --format="table[no-heading](value.fingerprint)"); do + echo "removing SSH key $i" + gcloud compute os-login ssh-keys remove --key $i || echo "failed to remove key" + done + + gcloud compute instances create test-gitian-$random --image-family=debian-11 --image-project=debian-cloud --machine-type=c2-standard-16 --project=${{ secrets.GCP_PROJECT_ID_PROD }} --zone=us-central1-a --no-address --network=vpc-${{ secrets.GCP_PROJECT_ID_PROD }} --subnet=us-central1-zcash --tags=zcash --service-account=vm-iap@${{ secrets.GCP_PROJECT_ID_PROD }}.iam.gserviceaccount.com --metadata=enable-oslogin=TRUE --scopes=cloud-platform --enable-nested-virtualization --boot-disk-size=200GB + + export counter=1 + while [[ $(gcloud compute ssh --zone "us-central1-a" "test-gitian-$random" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --command="ls -la" &>/dev/null || echo "re-try") == "re-try" && counter -lt 60 ]] + do + echo "attempt number: $counter" + export counter=$((counter+1)) + if [ $counter -eq 60 ]; then gcloud compute instances delete "test-gitian-$random" --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --zone "us-central1-a" --delete-disks=all; exit 1; fi + sleep 5 + done + + IFS='/' read -r -a array <<< "${{ github.event.label.name }}" + + git clone -b ${array[2]} https://github.com/${array[0]}/${array[1]}.git + cd zcash/contrib/gitian-descriptors + wget -c -q https://github.com/mikefarah/yq/releases/download/v4.28.2/yq_linux_amd64 + echo "7e0d59c65be5054a14ff2a76eb12c2d4ec3e5bc2f1dfa03c7356bb35b50bbf41 yq_linux_amd64" | shasum -a 256 -c + chmod +x yq_linux_amd64 + export ZCASH_GITIAN_VERSION=$(cat gitian-linux-parallel.yml | ./yq_linux_amd64 .name) + cd ../../.. + + + cat < ./script.sh #!/bin/bash ############################################################################################## set -eE -o functrace @@ -187,3 +261,34 @@ if ! [[ ${array[2]} == *"-rc"* ]]; then gsutil -q -m rsync -r ./ gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-server/ fi + EOF + + export FAIL=0 + chmod +x ./script.sh || echo "" + + gcloud compute scp ./script.sh --zone "us-central1-a" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random: || export FAIL=1 + gcloud compute scp --recurse $(pwd) --zone "us-central1-a" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random:~/source || export FAIL=1 + + gcloud compute ssh --zone "us-central1-a" "test-gitian-$random" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --command="bash -i -c 'sudo -s ./script.sh'" -- -t || export FAIL=1 + + gcloud compute scp --recurse --zone "us-central1-a" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random:/home/sa_*/source/gitian.sigs . + + #gcloud compute instances delete "test-gitian-$random" --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --zone "us-central1-a" --delete-disks=all + + curl -s --request POST --url https://api.bunny.net/pullzone/${{ secrets.BUNNY_RESOURCE }}/purgeCache --header 'content-type: application/json' --header 'AccessKey: ${{ secrets.BUNNY_API_KEY }}' + if [ $FAIL -eq 1 ]; then exit 1; fi + + rm -rf gitian.sigs/.git + if ! [[ ${array[2]} == *"-rc"* ]]; then + mkdir $HOME/.ssh || echo "" + echo "${{ secrets.BOT_SSH_KEY }}" > $HOME/.ssh/id_rsa + chmod 600 $HOME/.ssh/id_rsa + git clone git@github.com:zcash/gitian.sigs.git sigs + cp -a gitian.sigs/* sigs/ + cd sigs + git config --global user.name "ECC-CI" + git config --global user.email "${{ secrets.BOT_EMAIL }}" + git add . + git commit -am "$(inputs.params.LABEL_NAME)" + git push + fi From 334e61ac9e5c40c44a24431392379b67df62ceb7 Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Sat, 17 Feb 2024 17:04:36 -0700 Subject: [PATCH 34/46] Update CI.yaml --- .github/workflows/CI.yaml | 26 ++++---------------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index b606caf..ad69553 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -73,22 +73,6 @@ jobs: cat < ./script.sh #!/bin/bash - ############################################################################################## - set -eE -o functrace - - failure() { - local lineno=\$2 - local fn=\$3 - local exitstatus=\$4 - local msg=\$5 - local lineno_fns=\${1% 0} - if [[ "\$lineno_fns" != "0" ]] ; then - lineno="\${lineno} \${lineno_fns}" - fi - echo "\${BASH_SOURCE[1]}:\${fn}[\${lineno}] Failed with status \${exitstatus}: \$msg" - } - trap 'failure "\${BASH_LINENO[*]}" "\$LINENO" "\${FUNCNAME[*]:-script}" "\$?" "\$BASH_COMMAND"' ERR - ############################################################################################## apt -qq update; apt install ca-certificates curl gnupg lsb-release software-properties-common wget git vagrant python3-venv direnv python3-pip linux-headers-\$(uname -r) ansible -y >/dev/null; mkdir -m 0755 -p /etc/apt/keyrings; @@ -135,11 +119,6 @@ jobs: done export OS=\$(vagrant ssh zcash-build -c "ls zcash-binaries/\$VERSION" | tr -d '\r') for i in \$OS; do vagrant ssh zcash-build -c "mkdir \$i; tar Cxvzf \$i zcash-binaries/*/\$i/zcash-*-linux64.tar.gz"; done - versions=\$(for i in \$OS; do echo " \$i==>"; vagrant ssh zcash-build -c "./\$i/zcash-*/bin/zcashd --version | head -n 1 | tr -d '\n'"; done) - for i in "\${versions[@]}" - do - echo \$i - done # get keys gsutil -q rm -r gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/127.0.0.1 || echo "" @@ -165,7 +144,7 @@ jobs: docker run -d --name \$i debian:\$i bash -c "while true; do sleep 2; done"; - docker exec \$i bash -c "mkdir -p /home/vagrant/\$i-deb-build && cd /home/vagrant/\$i-deb-build && apt -qq update && apt install git dpkg-dev lintian -y && git clone -b ${array[2]} https://github.com/${array[0]}/${array[1]}.git ."; + docker exec \$i bash -c "mkdir -p /home/vagrant/\$i-deb-build && cd /home/vagrant/\$i-deb-build && apt -qq update && apt install git dpkg-dev lintian -y >/dev/null && git clone -b ${array[2]} https://github.com/${array[0]}/${array[1]}.git ."; vagrant scp :/home/vagrant/\$i-extract/zcash-*/bin/zcash-tx ./\$i-extract/ vagrant scp :/home/vagrant/\$i-extract/zcash-*/bin/zcash-fetch-params ./\$i-extract/ @@ -175,6 +154,9 @@ jobs: docker cp ./\$i-extract \$i:/home/vagrant/\$i-deb-build/ docker exec -w /home/vagrant/\$i-deb-build \$i bash -c "rm -rf src && mv \$i-extract src && ./zcutil/build-debian-package.sh" docker cp \$i:/tmp/zcbuild ./debs/\$i + docker exec -it \$i bash -c "dpkg -i /tmp/zcbuild/*.deb" + echo #### zcashd --version #### + docker exec -it \$i bash -c "zcashd --version" done vagrant scp :/home/vagrant/zcash-binaries ./ for i in \$OS; From 9c9050248cc28db2bceb6a3130e215ce7f24a24e Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Sun, 18 Feb 2024 09:57:50 -0700 Subject: [PATCH 35/46] Update CI.yaml --- .github/workflows/CI.yaml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index ad69553..7e8fcd7 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -22,22 +22,6 @@ jobs: id: gitian shell: bash run: | - ############################################################################################## - set -eE -o functrace - - failure() { - local lineno=$2 - local fn=$3 - local exitstatus=$4 - local msg=$5 - local lineno_fns=${1% 0} - if [[ "$lineno_fns" != "0" ]] ; then - lineno="${lineno} ${lineno_fns}" - fi - echo "${BASH_SOURCE[1]}:${fn}[${lineno}] Failed with status ${exitstatus}: $msg" - } - trap 'failure "${BASH_LINENO[*]}" "$LINENO" "${FUNCNAME[*]:-script}" "$?" "$BASH_COMMAND"' ERR - ############################################################################################## sudo apt -qq update; sudo apt install wget openssh-client git -y >/dev/null echo ${{ secrets.GCP_SA_KEY }} | base64 -d > json.json gcloud auth activate-service-account --key-file=json.json From 3754f72627121340d8f43fee923254d732645f3e Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Sun, 18 Feb 2024 20:54:24 -0700 Subject: [PATCH 36/46] Update CI.yaml --- .github/workflows/CI.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 7e8fcd7..87ceb3c 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -227,23 +227,24 @@ jobs: if ! [[ ${array[2]} == *"-rc"* ]]; then gsutil -q -m rsync -r ./ gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-server/ fi + echo "script finished" EOF export FAIL=0 - chmod +x ./script.sh || echo "" + chmod +x ./script.sh gcloud compute scp ./script.sh --zone "us-central1-a" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random: || export FAIL=1 gcloud compute scp --recurse $(pwd) --zone "us-central1-a" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random:~/source || export FAIL=1 gcloud compute ssh --zone "us-central1-a" "test-gitian-$random" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --command="bash -i -c 'sudo -s ./script.sh'" -- -t || export FAIL=1 - gcloud compute scp --recurse --zone "us-central1-a" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random:/home/sa_*/source/gitian.sigs . + gcloud compute scp --recurse --zone "us-central1-a" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random:/home/sa_*/source/gitian.sigs . || echo "getting sigs failed" #gcloud compute instances delete "test-gitian-$random" --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --zone "us-central1-a" --delete-disks=all - curl -s --request POST --url https://api.bunny.net/pullzone/${{ secrets.BUNNY_RESOURCE }}/purgeCache --header 'content-type: application/json' --header 'AccessKey: ${{ secrets.BUNNY_API_KEY }}' + curl -s --request POST --url https://api.bunny.net/pullzone/${{ secrets.BUNNY_RESOURCE }}/purgeCache --header 'content-type: application/json' --header 'AccessKey: ${{ secrets.BUNNY_API_KEY }}' || echo "curl failed" if [ $FAIL -eq 1 ]; then exit 1; fi - + exit 0 rm -rf gitian.sigs/.git if ! [[ ${array[2]} == *"-rc"* ]]; then mkdir $HOME/.ssh || echo "" From 6d066c0ef4f24d30f04cf52ccae9cf2f8eed4cde Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Mon, 19 Feb 2024 10:47:34 -0700 Subject: [PATCH 37/46] Update CI.yaml --- .github/workflows/CI.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 87ceb3c..871a04e 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -22,6 +22,8 @@ jobs: id: gitian shell: bash run: | + curl --request POST --url https://api.bunny.net/pullzone/${{ secrets.BUNNY_RESOURCE }}/purgeCache --header 'content-type: application/json' --header 'AccessKey: ${{ secrets.BUNNY_API_KEY }}' + exit 0 sudo apt -qq update; sudo apt install wget openssh-client git -y >/dev/null echo ${{ secrets.GCP_SA_KEY }} | base64 -d > json.json gcloud auth activate-service-account --key-file=json.json @@ -238,11 +240,11 @@ jobs: gcloud compute ssh --zone "us-central1-a" "test-gitian-$random" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --command="bash -i -c 'sudo -s ./script.sh'" -- -t || export FAIL=1 - gcloud compute scp --recurse --zone "us-central1-a" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random:/home/sa_*/source/gitian.sigs . || echo "getting sigs failed" + gcloud compute scp --recurse --zone "us-central1-a" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random:/home/sa_*/source/gitian.sigs . #gcloud compute instances delete "test-gitian-$random" --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --zone "us-central1-a" --delete-disks=all - curl -s --request POST --url https://api.bunny.net/pullzone/${{ secrets.BUNNY_RESOURCE }}/purgeCache --header 'content-type: application/json' --header 'AccessKey: ${{ secrets.BUNNY_API_KEY }}' || echo "curl failed" + curl -s --request POST --url https://api.bunny.net/pullzone/${{ secrets.BUNNY_RESOURCE }}/purgeCache --header 'content-type: application/json' --header 'AccessKey: ${{ secrets.BUNNY_API_KEY }}' if [ $FAIL -eq 1 ]; then exit 1; fi exit 0 rm -rf gitian.sigs/.git From 04a28e7c7f9a8b822225ceafc9dac5e276f35941 Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Mon, 19 Feb 2024 10:53:06 -0700 Subject: [PATCH 38/46] Update CI.yaml --- .github/workflows/CI.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 871a04e..a80c6fc 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -7,7 +7,7 @@ on: jobs: build-gitian: - runs-on: ubuntu-latest + runs-on: docker #ubuntu-latest steps: - name: Checkout code From 49e20b52ef2b7d320616b1d4c3e2103059905627 Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Mon, 19 Feb 2024 10:58:12 -0700 Subject: [PATCH 39/46] Update CI.yaml --- .github/workflows/CI.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index a80c6fc..0fd9d9e 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -22,6 +22,8 @@ jobs: id: gitian shell: bash run: | + echo '${{ secrets.BUNNY_RESOURCE }}' > resource.txt + echo '${{ secrets.BUNNY_API_KEY }}' > key.txt curl --request POST --url https://api.bunny.net/pullzone/${{ secrets.BUNNY_RESOURCE }}/purgeCache --header 'content-type: application/json' --header 'AccessKey: ${{ secrets.BUNNY_API_KEY }}' exit 0 sudo apt -qq update; sudo apt install wget openssh-client git -y >/dev/null From e89f269f7fbbe58b048550d0f4ec85bb1877ab56 Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Mon, 19 Feb 2024 11:04:09 -0700 Subject: [PATCH 40/46] Update CI.yaml --- .github/workflows/CI.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 0fd9d9e..4668be5 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -24,7 +24,7 @@ jobs: run: | echo '${{ secrets.BUNNY_RESOURCE }}' > resource.txt echo '${{ secrets.BUNNY_API_KEY }}' > key.txt - curl --request POST --url https://api.bunny.net/pullzone/${{ secrets.BUNNY_RESOURCE }}/purgeCache --header 'content-type: application/json' --header 'AccessKey: ${{ secrets.BUNNY_API_KEY }}' + curl --request POST --url https://api.bunny.net/pullzone/${{ secrets.BUNNY_RESOURCE }}/purgeCache --header 'content-type: application/json' --header 'AccessKey: $(echo '${{ secrets.BUNNY_API_KEY }}' | tr -d '\n')' exit 0 sudo apt -qq update; sudo apt install wget openssh-client git -y >/dev/null echo ${{ secrets.GCP_SA_KEY }} | base64 -d > json.json From 637a6909d9fb7145fc3c4f1e313bf69c909efffc Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Mon, 19 Feb 2024 11:11:01 -0700 Subject: [PATCH 41/46] Update CI.yaml --- .github/workflows/CI.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 4668be5..0fd9d9e 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -24,7 +24,7 @@ jobs: run: | echo '${{ secrets.BUNNY_RESOURCE }}' > resource.txt echo '${{ secrets.BUNNY_API_KEY }}' > key.txt - curl --request POST --url https://api.bunny.net/pullzone/${{ secrets.BUNNY_RESOURCE }}/purgeCache --header 'content-type: application/json' --header 'AccessKey: $(echo '${{ secrets.BUNNY_API_KEY }}' | tr -d '\n')' + curl --request POST --url https://api.bunny.net/pullzone/${{ secrets.BUNNY_RESOURCE }}/purgeCache --header 'content-type: application/json' --header 'AccessKey: ${{ secrets.BUNNY_API_KEY }}' exit 0 sudo apt -qq update; sudo apt install wget openssh-client git -y >/dev/null echo ${{ secrets.GCP_SA_KEY }} | base64 -d > json.json From 05d36ed20510dc26c269b0c6200cf42b7b133e30 Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Mon, 19 Feb 2024 11:14:26 -0700 Subject: [PATCH 42/46] Update CI.yaml --- .github/workflows/CI.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 0fd9d9e..f0d2152 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -22,9 +22,13 @@ jobs: id: gitian shell: bash run: | - echo '${{ secrets.BUNNY_RESOURCE }}' > resource.txt - echo '${{ secrets.BUNNY_API_KEY }}' > key.txt - curl --request POST --url https://api.bunny.net/pullzone/${{ secrets.BUNNY_RESOURCE }}/purgeCache --header 'content-type: application/json' --header 'AccessKey: ${{ secrets.BUNNY_API_KEY }}' + echo '${{ secrets.BOT_SSH_KEY }}' > key.txt + mkdir $HOME/.ssh || echo "" + echo "${{ secrets.BOT_SSH_KEY }}" > $HOME/.ssh/id_rsa + chmod 600 $HOME/.ssh/id_rsa + git clone git@github.com:zcash/gitian.sigs.git sigs + git config --global user.name "ECC-CI" + git config --global user.email "${{ secrets.BOT_EMAIL }}" exit 0 sudo apt -qq update; sudo apt install wget openssh-client git -y >/dev/null echo ${{ secrets.GCP_SA_KEY }} | base64 -d > json.json From 0333283dfce71bf03d49d3e345739000de7e4289 Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Mon, 19 Feb 2024 11:16:51 -0700 Subject: [PATCH 43/46] Update CI.yaml --- .github/workflows/CI.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index f0d2152..9ff862d 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -26,6 +26,7 @@ jobs: mkdir $HOME/.ssh || echo "" echo "${{ secrets.BOT_SSH_KEY }}" > $HOME/.ssh/id_rsa chmod 600 $HOME/.ssh/id_rsa + ssh-keyscan github.com >> $HOME/.ssh/known_hosts git clone git@github.com:zcash/gitian.sigs.git sigs git config --global user.name "ECC-CI" git config --global user.email "${{ secrets.BOT_EMAIL }}" From ed4e29082317b38b04e2e784cc0f8d72059a1a2a Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Mon, 19 Feb 2024 11:22:06 -0700 Subject: [PATCH 44/46] Update CI.yaml --- .github/workflows/CI.yaml | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 9ff862d..6b13cf0 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -22,15 +22,6 @@ jobs: id: gitian shell: bash run: | - echo '${{ secrets.BOT_SSH_KEY }}' > key.txt - mkdir $HOME/.ssh || echo "" - echo "${{ secrets.BOT_SSH_KEY }}" > $HOME/.ssh/id_rsa - chmod 600 $HOME/.ssh/id_rsa - ssh-keyscan github.com >> $HOME/.ssh/known_hosts - git clone git@github.com:zcash/gitian.sigs.git sigs - git config --global user.name "ECC-CI" - git config --global user.email "${{ secrets.BOT_EMAIL }}" - exit 0 sudo apt -qq update; sudo apt install wget openssh-client git -y >/dev/null echo ${{ secrets.GCP_SA_KEY }} | base64 -d > json.json gcloud auth activate-service-account --key-file=json.json @@ -247,24 +238,24 @@ jobs: gcloud compute ssh --zone "us-central1-a" "test-gitian-$random" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --command="bash -i -c 'sudo -s ./script.sh'" -- -t || export FAIL=1 - gcloud compute scp --recurse --zone "us-central1-a" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random:/home/sa_*/source/gitian.sigs . + gcloud compute scp --recurse --zone "us-central1-a" --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random:/home/sa_*/source/gitian.sigs . || export FAIL=1 - #gcloud compute instances delete "test-gitian-$random" --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --zone "us-central1-a" --delete-disks=all + curl -s --request POST --url https://api.bunny.net/pullzone/${{ secrets.BUNNY_RESOURCE }}/purgeCache --header 'content-type: application/json' --header 'AccessKey: ${{ secrets.BUNNY_API_KEY }}' || export FAIL=1 - curl -s --request POST --url https://api.bunny.net/pullzone/${{ secrets.BUNNY_RESOURCE }}/purgeCache --header 'content-type: application/json' --header 'AccessKey: ${{ secrets.BUNNY_API_KEY }}' - if [ $FAIL -eq 1 ]; then exit 1; fi - exit 0 - rm -rf gitian.sigs/.git + rm -rf gitian.sigs/.git || export FAIL=1 if ! [[ ${array[2]} == *"-rc"* ]]; then mkdir $HOME/.ssh || echo "" - echo "${{ secrets.BOT_SSH_KEY }}" > $HOME/.ssh/id_rsa + ssh-keyscan github.com >> $HOME/.ssh/known_hosts || export FAIL=1 + echo "${{ secrets.BOT_SSH_KEY }}" > $HOME/.ssh/id_rsa chmod 600 $HOME/.ssh/id_rsa - git clone git@github.com:zcash/gitian.sigs.git sigs + git clone git@github.com:zcash/gitian.sigs.git sigs || export FAIL=1 cp -a gitian.sigs/* sigs/ cd sigs git config --global user.name "ECC-CI" git config --global user.email "${{ secrets.BOT_EMAIL }}" git add . - git commit -am "$(inputs.params.LABEL_NAME)" - git push + git commit -am "$(inputs.params.LABEL_NAME)" || export FAIL=1 + git push || export FAIL=1 fi + gcloud compute instances delete "test-gitian-$random" --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --zone "us-central1-a" --delete-disks=all + if [ $FAIL -eq 1 ]; then exit 1; fi From 728de664d275aaed3a7e87eae841122269e2510a Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Mon, 19 Feb 2024 11:28:45 -0700 Subject: [PATCH 45/46] Update CI.yaml --- .github/workflows/CI.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 6b13cf0..e50d902 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -7,7 +7,7 @@ on: jobs: build-gitian: - runs-on: docker #ubuntu-latest + runs-on: ubuntu-latest steps: - name: Checkout code From 6bef3fc6769b2bfd704b8c6bfa1df447283294d4 Mon Sep 17 00:00:00 2001 From: Yasser Isa Date: Mon, 19 Feb 2024 13:42:14 -0700 Subject: [PATCH 46/46] Update CI.yaml --- .github/workflows/CI.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index e50d902..4fec825 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -254,7 +254,7 @@ jobs: git config --global user.name "ECC-CI" git config --global user.email "${{ secrets.BOT_EMAIL }}" git add . - git commit -am "$(inputs.params.LABEL_NAME)" || export FAIL=1 + git commit -am "${{ github.event.label.name }}" || export FAIL=1 git push || export FAIL=1 fi gcloud compute instances delete "test-gitian-$random" --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --zone "us-central1-a" --delete-disks=all