Mark git directories as safe for our docker_ci container and fix carg…

…o caching (#1187)

* Mark git directories as safe for our docker_ci container
* New cargo packages means a new cache key
* Add CACHE_KEY everywhere and Only update cache from 1 location

.github/workflows/python.yml

@@ -62,26 +62,26 @@ jobs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
 
     steps:
-    # Sets TAGGED_OR_MAIN if this workflow is running on a tag or the main branch.
-    - name: Set TAGGED_OR_MAIN
-      if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main'
-      run: echo "TAGGED_OR_MAIN=1" >> $GITHUB_ENV
+      # Sets TAGGED_OR_MAIN if this workflow is running on a tag or the main branch.
+      - name: Set TAGGED_OR_MAIN
+        if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main'
+        run: echo "TAGGED_OR_MAIN=1" >> $GITHUB_ENV
 
-    - id: set-matrix
-      shell: bash
-      run: |
-        matrix=()
+      - id: set-matrix
+        shell: bash
+        run: |
+          matrix=()
 
-        if [[ $TAGGED_OR_MAIN ]]; then
-          # MacOS build is really slow (>30 mins); uses up a lot of CI minutes
-          matrix+=('{"platform": "macos", "runs_on": "macos-latest"},')
-        fi
-        matrix+=('{"platform": "windows", "runs_on": "windows-latest-8-cores"},')
-        matrix+=('{"platform": "linux", "runs_on": "ubuntu-latest-16-cores"}')
+          if [[ $TAGGED_OR_MAIN ]]; then
+            # MacOS build is really slow (>30 mins); uses up a lot of CI minutes
+            matrix+=('{"platform": "macos", "runs_on": "macos-latest"},')
+          fi
+          matrix+=('{"platform": "windows", "runs_on": "windows-latest-8-cores"},')
+          matrix+=('{"platform": "linux", "runs_on": "ubuntu-latest-16-cores"}')
 
-        echo "Matrix values: ${matrix[@]}"
+          echo "Matrix values: ${matrix[@]}"
 
-        echo "matrix={\"include\":[${matrix[@]}]}" >> $GITHUB_OUTPUT
+          echo "matrix={\"include\":[${matrix[@]}]}" >> $GITHUB_OUTPUT
 
   wheels:
     name: Build Python Wheels
@@ -108,8 +108,11 @@ jobs:
       - name: Set up cargo cache
         uses: Swatinem/rust-cache@v2
         with:
-          # See: https://github.com/rerun-io/rerun/pull/497
-          save-if: ${{ github.event_name == 'push'}}
+          env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY
+          # Don't update the cache -- it will be updated by the lint job
+          # TODO(jleibs): this job will likely run before rust.yml updates
+          # the cache. Better cross-job sequencing would be nice here
+          save-if: False
 
       - name: Setup python
         uses: actions/setup-python@v4

.github/workflows/rust.yml

@@ -45,7 +45,10 @@ jobs:
         uses: Swatinem/rust-cache@v2
         with:
           env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY
-          save-if: ${{ github.event_name == 'push'}}
+          # Don't update the cache -- it will be updated by the lint job
+          # TODO(jleibs): this job will likely run before rust.yml updates
+          # the cache. Better cross-job sequencing would be nice here
+          save-if: false
 
       - name: Run benchmark
         # Use bash shell so we get pipefail behavior with tee

.github/workflows/toml.yml

@@ -27,7 +27,11 @@ jobs:
         uses: Swatinem/rust-cache@v2
         with:
           # See: https://github.com/rerun-io/rerun/pull/497
-          save-if: ${{ github.event_name == 'push'}}
+          env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY
+          # Don't update the cache -- it will be updated by the lint job
+          # TODO(jleibs): this job will likely run before rust.yml updates
+          # the cache. Better cross-job sequencing would be nice here
+          save-if: false
 
       - name: Install taplo-cli
         uses: baptiste0928/cargo-install@v1

ci_docker/Dockerfile

@@ -1,6 +1,8 @@
 FROM rust:1.67
 LABEL maintainer="[email protected]"
-LABEL version="0.1"
+# Remember to update the version in publish.sh
+# TODO(jleibs) use this version in the publish.sh script and below in the CACHE_KEY
+LABEL version="0.4"
 LABEL description="Docker image used for the CI of https://github.com/rerun-io/rerun"
 
 # Install some cargo tools we know we'll always need
@@ -34,4 +36,7 @@ RUN pip install -r requirements-build.txt
 RUN cargo install [email protected]
 
 # Increment this to invalidate cache
-ENV CACHE_KEY=rerun_docker_v0.1
+ENV CACHE_KEY=rerun_docker_v0.4
+
+# See: https://github.com/actions/runner-images/issues/6775#issuecomment-1410270956
+RUN git config --system --add safe.directory '*'

ci_docker/publish.sh

@@ -1,4 +1,4 @@
-VERSION=0.4 # Bump on each new version. Maybe it should also match the version in ci_docker/Dockerfile ?
+VERSION=0.4 # Bump on each new version. Remember to update the version in the Dockerfile too.
 
 # The build needs to run from top of repo to access the requirments.txt
 cd `git rev-parse --show-toplevel`
@@ -7,8 +7,10 @@ cd `git rev-parse --show-toplevel`
 docker pull rerunio/ci_docker
 
 # Build the image
-# docker build -t ci_docker -f ci_docker/Dockerfile .
-docker buildx build --platform=linux/amd64 -t ci_docker -f ci_docker/Dockerfile .
+docker build -t ci_docker -f ci_docker/Dockerfile .
+# This is necessary to build on mac, but is doing something weird with the Cache
+# TODO(jleibs): Make this all work portably with caching
+# docker buildx build --platform=linux/amd64 -t ci_docker -f ci_docker/Dockerfile .
 
 # Tag latest and version
 docker tag ci_docker rerunio/ci_docker