Bookworm release + Github Action CI

.github/workflows/CI.yaml

@@ -0,0 +1,261 @@
+name: Gitian CI
+
+on:
+  pull_request:
+    types:
+      - labeled
+
+jobs:
+  build-gitian:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    - name: 'Set up Cloud SDK'
+      uses: 'google-github-actions/setup-gcloud@v2'
+      with:
+        version: '>= 363.0.0'
+
+    - name: Build Gitian
+      id: gitian
+      shell: bash
+      run: |
+        sudo apt -qq update; sudo apt install wget openssh-client git -y >/dev/null
+        echo ${{ secrets.GCP_SA_KEY }} | base64 -d > json.json
+        gcloud auth activate-service-account --key-file=json.json
+
+        export random=$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 12; echo;)
+
+        for i in $(gcloud compute os-login ssh-keys list --format="table[no-heading](value.fingerprint)"); do 
+          echo "removing SSH key $i"
+          gcloud compute os-login ssh-keys remove --key $i || echo "failed to remove key"
+        done
+
+        gcloud compute instances create test-gitian-$random --image-family=debian-11 --image-project=debian-cloud --machine-type=c2-standard-16 --project=${{ secrets.GCP_PROJECT_ID_PROD }} --zone=us-central1-a --no-address --network=vpc-${{ secrets.GCP_PROJECT_ID_PROD }} --subnet=us-central1-zcash --tags=zcash --service-account=vm-iap@${{ secrets.GCP_PROJECT_ID_PROD }}.iam.gserviceaccount.com --metadata=enable-oslogin=TRUE --scopes=cloud-platform --enable-nested-virtualization --boot-disk-size=200GB
+
+        export counter=1
+        while [[ $(gcloud compute ssh --zone "us-central1-a" "test-gitian-$random"  --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --command="ls -la" &>/dev/null || echo "re-try") == "re-try" && counter -lt 60 ]]
+        do
+          echo "attempt number: $counter"
+          export counter=$((counter+1))
+          if [ $counter -eq 60 ]; then gcloud compute instances delete "test-gitian-$random" --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --zone "us-central1-a" --delete-disks=all; exit 1; fi
+          sleep 5
+        done
+
+        IFS='/' read -r -a array <<< "${{ github.event.label.name }}"
+
+        git clone -b ${array[2]} https://github.com/${array[0]}/${array[1]}.git
+        cd zcash/contrib/gitian-descriptors
+        wget -c -q https://github.com/mikefarah/yq/releases/download/v4.28.2/yq_linux_amd64
+        echo "7e0d59c65be5054a14ff2a76eb12c2d4ec3e5bc2f1dfa03c7356bb35b50bbf41  yq_linux_amd64" | shasum -a 256 -c
+        chmod +x yq_linux_amd64
+        export ZCASH_GITIAN_VERSION=$(cat gitian-linux-parallel.yml | ./yq_linux_amd64 .name)
+        cd ../../..
+
+
+        cat <<EOF > ./script.sh
+        #!/bin/bash
+        apt -qq update;
+        apt install ca-certificates curl gnupg lsb-release software-properties-common wget git vagrant python3-venv direnv python3-pip linux-headers-\$(uname -r) ansible -y >/dev/null;
+        mkdir -m 0755 -p /etc/apt/keyrings;
+        curl -fsSL https://download.docker.com/linux/debian/gpg -o gpg.asc
+        echo "1500c1f56fa9e26b9b8f42452a553675796ade0807cdce11975eb98170b3a570  gpg.asc" | shasum -a 256 -c;
+        sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg --yes < gpg.asc;
+        echo "deb [arch=\$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \$(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+        apt -qq update;
+        apt install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y >/dev/null;
+        apt-add-repository "deb http://download.virtualbox.org/virtualbox/debian \$(lsb_release -sc) contrib";
+        wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc;
+        echo "49e6801d45f6536232c11be6cdb43fa8e0198538d29d1075a7e10165e1fbafe2  oracle_vbox_2016.asc" | shasum -a 256 -c;
+        apt-key add oracle_vbox_2016.asc;
+        apt -qq update
+        apt install virtualbox-6.1 -y >/dev/null;
+        eval "\$(direnv hook bash)";
+        cd source
+        cp .env.example .env
+        cp .envrc.example .envrc
+        /usr/bin/python3 -m venv ./local/python_venv;
+        echo "load_prefix local/python_venv" >> .envrc;
+        export VERSION="${array[2]}"
+        echo "ZCASH_VERSION=\$VERSION" >> .env;
+        echo "ZCASH_GIT_REPO_URL=https://github.com/${array[0]}/${array[1]}" >> .env;
+        cat .env
+        direnv allow;
+        /sbin/vboxconfig;
+        vagrant plugin install --local;
+        vagrant plugin install --local;
+        gpg --quick-generate-key --batch --passphrase '' "Lyra Silvertongue (zcash gitian) <[email protected]>"
+        echo "GPG_KEY_ID=\$(gpg --list-keys --with-fingerprint --with-colons | grep fpr: | head -n 1 | sed 's/fpr://g' | sed 's/://g')" >> .env;
+        echo "GPG_KEY_NAME=lyra.silvertongue" >> .env;
+        git config --global user.name "Lyra Silvertongue"
+        git config --global user.email "[email protected]"
+        direnv allow;
+        direnv exec \$(pwd) vagrant up zcash-build;
+        vagrant ssh zcash-build -c "gpg --quick-generate-key --batch --passphrase '' \"Lyra Silvertongue (zcash gitian) <[email protected]>\" || echo ''"
+        vagrant ssh zcash-build -c ./gitian-parallel-build.sh || exit 1
+        vagrant ssh zcash-build -c "head -n 8 gitian.sigs/\$VERSION*/lyra.silvertongue/*.assert" > assert.txt
+        tr -d \$'\r' < assert.txt > assert2.txt
+        echo "#### sigs ####"
+        for i in \$(cat assert2.txt | grep -E "zcash-*"  | grep -v git: | sed 's/    //g' | sed 's/  /-->/g'); do
+          echo \$i
+        done
+        export OS=\$(vagrant ssh zcash-build -c "ls zcash-binaries/\$VERSION" | tr -d '\r')
+        for i in \$OS; do vagrant ssh zcash-build -c "mkdir \$i; tar Cxvzf \$i zcash-binaries/*/\$i/zcash-*-linux64.tar.gz"; done
+
+        # get keys
+        gsutil -q rm -r gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/127.0.0.1 || echo ""
+        gsutil -q cp gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/encrypted_gpg.kms \$HOME/encrypted_gpg.kms
+        gsutil -q cp gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/public.asc \$HOME/public.asc
+        current_dir=\$(pwd)
+        cd \$HOME
+        gcloud kms decrypt \
+            --key gpg \
+            --keyring gpg \
+            --location global  \
+            --plaintext-file private.pgp \
+            --ciphertext-file encrypted_gpg.kms
+        cd \$current_dir
+        gpg --import \$HOME/private.pgp
+        vagrant scp :gitian.sigs .
+        for i in \$OS;
+        do
+          mkdir -p debs/\$i;
+          mkdir -p ./\$i-extract
+          vagrant ssh zcash-build -c "mkdir /home/vagrant/"\$i"-extract";
+          vagrant ssh zcash-build -c "tar -xvf /home/vagrant/zcash-binaries/"\$VERSION"/"\$i"/zcash-*-linux64.tar.gz -C /home/vagrant/"\$i"-extract";
+
+          docker run -d --name \$i debian:\$i bash -c "while true; do sleep 2; done";
+
+          docker exec \$i bash -c "mkdir -p /home/vagrant/\$i-deb-build && cd /home/vagrant/\$i-deb-build && apt -qq update && apt install git dpkg-dev lintian -y >/dev/null && git clone -b ${array[2]} https://github.com/${array[0]}/${array[1]}.git .";
+
+          vagrant scp :/home/vagrant/\$i-extract/zcash-*/bin/zcash-tx ./\$i-extract/
+          vagrant scp :/home/vagrant/\$i-extract/zcash-*/bin/zcash-fetch-params ./\$i-extract/
+          vagrant scp :/home/vagrant/\$i-extract/zcash-*/bin/zcashd ./\$i-extract/
+          vagrant scp :/home/vagrant/\$i-extract/zcash-*/bin/zcash-cli ./\$i-extract/
+          vagrant scp :/home/vagrant/\$i-extract/zcash-*/bin/zcashd-wallet-tool ./\$i-extract/
+          docker cp ./\$i-extract \$i:/home/vagrant/\$i-deb-build/
+          docker exec -w /home/vagrant/\$i-deb-build \$i bash -c "rm -rf src && mv \$i-extract src && ./zcutil/build-debian-package.sh"
+          docker cp \$i:/tmp/zcbuild ./debs/\$i
+          docker exec -it \$i bash -c "dpkg -i /tmp/zcbuild/*.deb"
+          echo #### zcashd --version #### 
+          docker exec -it \$i bash -c "zcashd --version"
+        done
+        vagrant scp :/home/vagrant/zcash-binaries ./
+        for i in \$OS;
+        do
+          cd ./zcash-binaries/\$VERSION/\$i
+          for j in \$(ls *linux64.tar.gz); do
+            mv \$j \$(echo \$j | sed 's/.tar.gz/-debian-'\$i'.tar.gz/g')
+          done
+          for j in \$(ls *debug.tar.gz); do
+            mv \$j \$(echo \$j | sed 's/.tar.gz/-debian-'\$i'.tar.gz/g')
+          done
+          gpg -u [email protected] --armor --digest-algo SHA256 --detach-sign *debug-debian-\$i.tar.gz
+          gpg -u [email protected] --armor --digest-algo SHA256 --detach-sign *linux64-debian-\$i.tar.gz
+          cd \$current_dir
+        done
+        export final_version=\$(cat assert2.txt | awk '{print \$2}' | grep "desc.yml" | head -n 1 | sed 's/-desc.yml//g')
+        gsutil -q -m rsync -r ./debs gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/debs
+        gsutil -q -m rsync -r ./zcash-binaries gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/zcash-binaries
+        apt install aptly -y >/dev/null
+
+        # generate apt
+        mkdir aptserver
+        cd aptserver
+        gsutil -q -m cp -r gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-server/pool/main/z/zcash/ .
+        cd zcash
+        cp -a ../../debs/buster/zcbuild/*.deb \$final_version-amd64-buster.deb
+        cp -a ../../debs/bullseye/zcbuild/*.deb \$final_version-amd64-bullseye.deb
+        cp -a ../../debs/bookworm/zcbuild/*.deb \$final_version-amd64-bookworm.deb
+
+        ls \$final_version-amd64-buster.deb || exit 1
+        ls \$final_version-amd64-bullseye.deb || exit 1
+        ls \$final_version-amd64-bookworm.deb || exit 1
+
+        aptly repo create --distribution buster --comment "" --component main zcash_buster_amd64_repo
+        aptly repo create --distribution bullseye --comment "" --component main zcash_bullseye_amd64_repo
+        aptly repo create --distribution bookworm --comment "" --component main zcash_bookworm_amd64_repo
+        aptly repo create --distribution stretch --comment "" --component main zcash_stretch_amd64_repo
+        for i in \$(ls *.deb | grep buster); do
+          aptly repo add zcash_buster_amd64_repo \$i
+        done
+        for i in \$(ls *.deb | grep bullseye); do
+          aptly repo add zcash_bullseye_amd64_repo \$i
+        done
+        for i in \$(ls *.deb | grep stretch); do
+          aptly repo add zcash_stretch_amd64_repo \$i
+        done
+        for i in \$(ls *.deb | grep bookworm); do
+          aptly repo add zcash_bookworm_amd64_repo \$i
+        done
+        aptly snapshot create bookworm_snapshot from repo zcash_bookworm_amd64_repo
+        aptly snapshot create buster_snapshot from repo zcash_buster_amd64_repo
+        aptly snapshot create bullseye_snapshot from repo zcash_bullseye_amd64_repo
+        aptly snapshot create stretch_snapshot from repo zcash_stretch_amd64_repo
+
+        export key=\$(gpg --list-secret-keys --keyid-format=long [email protected] | head -n 2 | grep -v sec)
+        aptly publish snapshot --distribution buster --component main --architectures amd64 --gpg-key="\$key" --passphrase="" buster_snapshot
+        aptly publish snapshot --distribution bookworm --component main --architectures amd64 --gpg-key="\$key" --passphrase="" bookworm_snapshot
+        aptly publish snapshot --distribution bullseye --component main --architectures amd64 --gpg-key="\$key" --passphrase="" bullseye_snapshot
+        aptly publish snapshot --distribution stretch --component main --architectures amd64 --gpg-key="\$key" --passphrase="" stretch_snapshot
+
+        apt install nginx-extras -y >/dev/null
+        cat << EOH > /etc/nginx/sites-enabled/default
+        server {
+                listen 80 default_server;
+                root /var/www/public;
+                location / {
+                    autoindex on;
+                }
+                server_name _;
+        }
+        EOH
+        # get apt server
+        cp -a /root/.aptly/public /var/www/
+        chown -R www-data:www-data /var/www
+        /etc/init.d/nginx restart
+        mkdir \$HOME/mirror
+        cd \$HOME/mirror
+        wget -q -r 127.0.0.1
+
+        cp \$HOME/public.asc \$HOME/mirror/127.0.0.1/zcash.asc
+        cd \$HOME/mirror
+        gsutil -q -m rsync -r ./127.0.0.1 gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/127.0.0.1
+        cd 127.0.0.1
+        if ! [[ ${array[2]} == *"-rc"* ]]; then
+          gsutil -q -m rsync -r ./ gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-server/
+        fi
+        echo "script finished"
+        EOF
+
+        export FAIL=0
+        chmod +x ./script.sh
+
+        gcloud compute scp ./script.sh --zone "us-central1-a"  --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random: || export FAIL=1
+        gcloud compute scp --recurse $(pwd) --zone "us-central1-a"  --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random:~/source || export FAIL=1
+
+        gcloud compute ssh --zone "us-central1-a" "test-gitian-$random"  --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --command="bash -i -c 'sudo -s ./script.sh'" -- -t || export FAIL=1
+
+        gcloud compute scp --recurse --zone "us-central1-a"  --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random:/home/sa_*/source/gitian.sigs . || export FAIL=1
+
+        curl -s --request POST --url https://api.bunny.net/pullzone/${{ secrets.BUNNY_RESOURCE }}/purgeCache --header 'content-type: application/json' --header 'AccessKey: ${{ secrets.BUNNY_API_KEY }}' || export FAIL=1
+
+        rm -rf gitian.sigs/.git || export FAIL=1
+        if ! [[ ${array[2]} == *"-rc"* ]]; then
+          mkdir $HOME/.ssh || echo ""
+          ssh-keyscan github.com >> $HOME/.ssh/known_hosts || export FAIL=1
+          echo "${{ secrets.BOT_SSH_KEY }}" > $HOME/.ssh/id_rsa 
+          chmod 600 $HOME/.ssh/id_rsa
+          git clone [email protected]:zcash/gitian.sigs.git sigs || export FAIL=1
+          cp -a gitian.sigs/* sigs/
+          cd sigs
+          git config --global user.name "ECC-CI"
+          git config --global user.email "${{ secrets.BOT_EMAIL }}"
+          git add .
+          git commit -am "${{ github.event.label.name }}" || export FAIL=1
+          git push || export FAIL=1
+        fi
+        gcloud compute instances delete "test-gitian-$random" --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --zone "us-central1-a" --delete-disks=all
+        if [ $FAIL -eq 1 ]; then exit 1; fi 

roles/gitian/templates/gitian-build.sh

@@ -215,7 +215,11 @@ then
             echo ""
             echo "Compiling variant: ${VERSION}_${suite}"
             echo ""
-
+            #workaround python and python3 in buster
+            if [[ $suite = "buster" ]]
+            then
+                sed -i -e 's/- "python3"/- "python"/g' -e 's/- "python-is-python3"//g' ${suite_dir_path}/gitian-linux-parallel.yml;
+            fi
             ./bin/gbuild --fetch-tags -j ${proc} -m ${mem} --commit zcash=${COMMIT} --url zcash=${url} ${suite_dir_path}/gitian-linux.yml
             ./bin/gsign -p "$signProg" --signer "$SIGNER" --release ${VERSION}_${suite} --destination ${gitian_sigs_repo_path}/ ${suite_dir_path}/gitian-linux.yml
 

roles/gitian/templates/gitian-parallel-build.sh

@@ -215,7 +215,11 @@ then
             echo ""
             echo "Compiling variant: ${VERSION}_${suite}"
             echo ""
-
+            #workaround python and python3 in buster
+            if [[ $suite = "buster" ]]
+            then
+                sed -i -e 's/- "python3"/- "python"/g' -e 's/- "python-is-python3"//g' ${suite_dir_path}/gitian-linux-parallel.yml;
+            fi
             ./bin/gbuild --fetch-tags -j ${proc} -m ${mem} --commit zcash=${COMMIT} --url zcash=${url} ${suite_dir_path}/gitian-linux-parallel.yml
             ./bin/gsign -p "$signProg" --signer "$SIGNER" --release ${VERSION}_${suite} --destination ${gitian_sigs_repo_path}/ ${suite_dir_path}/gitian-linux-parallel.yml