Add Submodule Caching to Toolchain Generation #555
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| branches: | |
| - master | |
| jobs: | |
| cache: | |
| name: Update Submodule Cache | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| key: submodules-${{ steps.submodule-hash.outputs.HASH }} | |
| env: | |
| cache-path: | | |
| .git/modules | |
| binutils | |
| gdb | |
| gcc | |
| llvm | |
| newlib | |
| glibc | |
| musl | |
| uclib-ng | |
| dejagnu | |
| pk | |
| qemu | |
| spike | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Generate Submodule Hash | |
| id: submodule-hash | |
| run: echo "HASH=$(git submodule | sha1sum | sed 's/.\{3\}$//')" >> $GITHUB_OUTPUT | |
| - name: Check is Cache Exists for Exact Submodule Configuration | |
| id: cache-check | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.cache-path }} | |
| key: submodules-${{ steps.submodule-hash.outputs.HASH }} | |
| lookup-only: true | |
| - name: If no Cache Hit, Update Cache | |
| uses: actions/cache@v4 | |
| if: steps.cache-check.outputs.cache-hit != 'true' | |
| with: | |
| path: ${{ env.cache-path }} | |
| key: submodules-${{ steps.submodule-hash.outputs.HASH }} | |
| restore-keys: | | |
| submodules- | |
| - name: Clone submodules | |
| if: steps.cache-check.outputs.cache-hit != 'true' | |
| run: | | |
| git submodule update --init --progress --depth 1 --jobs $(nproc) binutils gdb gcc llvm newlib glibc musl dejagnu pk qemu spike | |
| git submodule update --init --progress uclibc-ng | |
| build: | |
| name: Build Toolchain Variants | |
| runs-on: ${{ matrix.os }} | |
| needs: [cache] | |
| env: | |
| cache-key: ${{ needs.cache.outputs.key }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-22.04, ubuntu-24.04] | |
| mode: [newlib, linux, musl, uclibc] | |
| target: [rv32gc-ilp32d, rv64gc-lp64d] | |
| compiler: [gcc, llvm] | |
| exclude: | |
| - mode: musl | |
| compiler: llvm | |
| - mode: uclibc | |
| compiler: llvm | |
| steps: | |
| - name: Remove unneeded frameworks to recover disk space | |
| run: | | |
| echo "-- Before --" | |
| df -h | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /usr/local/lib/android | |
| echo "-- After --" | |
| df -h | |
| - name: Generate Submodules List | |
| id: cache-path | |
| run: | | |
| submodules="gdb:binutils" | |
| case "${{ matrix.mode }}" in | |
| "linux") | |
| submodules="$submodules:glibc";; | |
| "musl") | |
| submodules="$submodules:musl";; | |
| "uclibc") | |
| submodules="$submodules:uclibc-ng";; | |
| "newlib") | |
| submodules="$submodules:newlib";; | |
| *) | |
| echo "Invalid Mode"; exit 1;; | |
| esac | |
| case "${{ matrix.compiler }}" in | |
| "gcc") | |
| submodules="$submodules:gcc";; | |
| "llvm") | |
| submodules="$submodules:llvm";; | |
| *) | |
| echo "Invalid Compiler"; exit 1;; | |
| esac | |
| submodules=$(echo $submodules | sed 's/:/\n/g') | |
| submodules=$submodules$'\n'$(echo "$submodules" | sed -e 's/^/.git\/modules\//') | |
| echo 'submodules<<EOF' >> $GITHUB_OUTPUT | |
| echo "$submodules" >> $GITHUB_OUTPUT | |
| echo 'EOF' >> $GITHUB_OUTPUT | |
| - uses: actions/checkout@v4 | |
| - name: Load Cache | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: | | |
| ${{ steps.cache-path.outputs.submodules }} | |
| key: ${{ env.cache-key }} | |
| - name: Install Dependencies | |
| run: sudo ./.github/setup-apt.sh | |
| - name: Build Toolchain | |
| run: | | |
| TARGET_TUPLE=($(echo ${{ matrix.target }} | tr "-" "\n")) | |
| BUILD_TOOLCHAIN="./configure --prefix=/opt/riscv --with-arch=${TARGET_TUPLE[0]} --with-abi=${TARGET_TUPLE[1]}" | |
| if [ "${{ matrix.compiler }}" == "llvm" ]; then # build toolchain with llvm | |
| $BUILD_TOOLCHAIN --enable-llvm | |
| else | |
| $BUILD_TOOLCHAIN | |
| fi | |
| sudo make -j $(nproc) ${{ matrix.mode }} | |
| - name: Generate Report | |
| if: | | |
| matrix.os == 'ubuntu-24.04' | |
| && (matrix.mode == 'linux' || matrix.mode == 'newlib') | |
| && matrix.compiler == 'gcc' | |
| run: | | |
| sudo make report-${{ matrix.mode }} -j $(nproc) | |
| - name: Recover Space | |
| run: | | |
| sudo du -hs / 2> /dev/null || true | |
| sudo rm -rf binutils dejagnu gcc gdb glibc llvm musl newlib pk qemu spike uclibc-ng || true | |
| sudo du -hs / 2> /dev/null || true | |
| - name: Tar Toolchain | |
| run: tar czvf riscv.tar.gz -C /opt/ riscv/ | |
| - name: Generate Prebuilt Toolchain Name | |
| id: toolchain-name-generator | |
| run: | | |
| if [[ "${{ matrix.target }}" == *"32"* ]]; then BITS=32; else BITS=64; fi | |
| case "${{ matrix.mode }}" in | |
| "linux") | |
| MODE="glibc";; | |
| "musl") | |
| MODE="musl";; | |
| "uclibc") | |
| MODE="uclibc-ng";; | |
| *) | |
| MODE="elf";; | |
| esac | |
| echo "TOOLCHAIN_NAME=riscv$BITS-$MODE-${{ matrix.os }}-${{ matrix.compiler }}-nightly" >> $GITHUB_OUTPUT | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ steps.toolchain-name-generator.outputs.TOOLCHAIN_NAME }} | |
| path: riscv.tar.gz | |
| test-sim: | |
| name: Test Simulation | |
| runs-on: ${{ matrix.os }} | |
| needs: [cache] | |
| env: | |
| cache-key: ${{ needs.cache.outputs.key }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-24.04] | |
| mode: [newlib] | |
| target: [rv64gc-lp64d] | |
| sim: [spike] | |
| steps: | |
| - name: Remove unneeded frameworks to recover disk space | |
| run: | | |
| echo "-- Before --" | |
| df -h | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /usr/local/lib/android | |
| echo "-- After --" | |
| df -h | |
| - uses: actions/checkout@v4 | |
| - uses: actions/cache/restore@v4 | |
| with: | |
| path: | | |
| .git/modules/gcc | |
| .git/modules/newlib | |
| .git/modules/binutils | |
| .git/modules/gdb | |
| .git/modules/spike | |
| .git/modules/pk | |
| .git/modules/dejagnu | |
| gcc | |
| newlib | |
| binutils | |
| gdb | |
| spike | |
| pk | |
| dejagnu | |
| key: ${{ env.cache-key }} | |
| - name: Install Dependencies | |
| run: sudo ./.github/setup-apt.sh | |
| - name: Build Toolchain | |
| run: | | |
| TARGET_TUPLE=($(echo ${{ matrix.target }} | tr "-" "\n")) | |
| ./configure --prefix=/opt/riscv --with-arch=${TARGET_TUPLE[0]} --with-abi=${TARGET_TUPLE[1]} --with-sim=${{ matrix.sim }} | |
| make -j $(nproc) ${{ matrix.mode }} | |
| - name: Generate Report | |
| run: make report-${{ matrix.mode }} -j $(nproc) | |
| build-multilib: | |
| if: ${{ false }} # Disable until multilib errors are triaged | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-24.04] | |
| mode: [newlib, linux] | |
| target: [rv64gc-lp64d] | |
| steps: | |
| - name: Remove unneeded frameworks to recover disk space | |
| run: | | |
| echo "-- Before --" | |
| df -h | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /usr/local/lib/android | |
| echo "-- After --" | |
| df -h | |
| - uses: actions/checkout@v4 | |
| - name: install dependencies | |
| run: sudo ./.github/setup-apt.sh | |
| - name: build toolchain | |
| run: | | |
| TARGET_TUPLE=($(echo ${{ matrix.target }} | tr "-" "\n")) | |
| ./configure --prefix=/opt/riscv --with-arch=${TARGET_TUPLE[0]} --with-abi=${TARGET_TUPLE[1]} --enable-multilib | |
| sudo make -j $(nproc) ${{ matrix.mode }} | |
| - name: make report | |
| run: | | |
| sudo make report-${{ matrix.mode }} -j $(nproc) | |
| - name: tarball build | |
| run: tar czvf riscv.tar.gz -C /opt/ riscv/ | |
| - name: generate prebuilt toolchain name | |
| id: toolchain-name-generator | |
| run: | | |
| if [[ "${{ matrix.target }}" == *"32"* ]]; then BITS=32; else BITS=64; fi | |
| case "${{ matrix.mode }}" in | |
| "linux") | |
| MODE="glibc";; | |
| "musl") | |
| MODE="musl";; | |
| "uclibc") | |
| MODE="uclibc-ng";; | |
| *) | |
| MODE="elf";; | |
| esac | |
| echo "TOOLCHAIN_NAME=riscv$BITS-$MODE-${{ matrix.os }}-multilib-nightly" >> $GITHUB_OUTPUT | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ steps.toolchain-name-generator.outputs.TOOLCHAIN_NAME }} | |
| path: riscv.tar.gz |