Merge pull request #3428 from dependabot/feelepxyz/add-buildkit-cachi…

…ng-to-ci-image

CI: Use buildkit caching when building Dockerfile.ci

.github/workflows/ci.yml

@@ -44,6 +44,7 @@ jobs:
           echo "BASE_IMAGE=ubuntu:18.04" >> $GITHUB_ENV
           echo "CORE_IMAGE=dependabot/dependabot-core" >> $GITHUB_ENV
           echo "CORE_CI_IMAGE=dependabot/dependabot-core-ci" >> $GITHUB_ENV
+          echo "CODE_DIR=/home/dependabot/dependabot-core" >> $GITHUB_ENV
       - name: Log in to Docker registry
         run: |
           if [ -n "${{ secrets.DOCKER_USERNAME }}" ] && [ -n "${{ secrets.DOCKER_PASSWORD }}" ]; then
@@ -58,40 +59,162 @@ jobs:
       - name: Pull Docker base images & warm Docker cache
         run: |
           docker pull "$BASE_IMAGE"
-          docker pull "$CORE_CI_IMAGE:core--$BRANCH_REF" ||
-            docker pull "$CORE_IMAGE:latest"
-          docker pull "$CORE_CI_IMAGE:ci--$BRANCH_REF" ||
-            "$CORE_CI_IMAGE:latest" || true
+          docker pull "$CORE_IMAGE:branch--$BRANCH_REF" ||
+            docker pull "$CORE_IMAGE:latest" || true
+          docker pull "$CORE_CI_IMAGE:branch--$BRANCH_REF" ||
+            docker pull "$CORE_CI_IMAGE:latest" || true
       - name: Build dependabot-core image
         run: |
           DOCKER_BUILDKIT=1 docker build \
             -t "$CORE_IMAGE:latest" \
-            -t "$CORE_CI_IMAGE:core--$BRANCH_REF" \
+            -t "$CORE_IMAGE:branch--$BRANCH_REF" \
             --build-arg BUILDKIT_INLINE_CACHE=1 \
             --cache-from "$BASE_IMAGE" \
-            --cache-from "$CORE_CI_IMAGE:core--$BRANCH_REF" \
+            --cache-from "$CORE_IMAGE:branch--$BRANCH_REF" \
             --cache-from "$CORE_IMAGE:latest" \
             .
       - name: Push dependabot-core image to Docker registry
         if: env.DOCKER_LOGGED_IN == 'true'
         run: |
-          docker push "$CORE_CI_IMAGE:core--$BRANCH_REF"
+          docker push "$CORE_IMAGE:branch--$BRANCH_REF"
       - name: Build dependabot-core-ci image
         run: |
-          rm .dockerignore
-          docker build \
+          DOCKER_BUILDKIT=1 docker build \
             -t "$CORE_CI_IMAGE:latest" \
-            -t "$CORE_CI_IMAGE:ci--$BRANCH_REF" \
+            -t "$CORE_CI_IMAGE:branch--$BRANCH_REF" \
             -f Dockerfile.ci \
+            --build-arg BUILDKIT_INLINE_CACHE=1 \
             --cache-from "$CORE_CI_IMAGE:latest" \
-            --cache-from "$CORE_CI_IMAGE:ci--$BRANCH_REF" \
+            --cache-from "$CORE_CI_IMAGE:branch--$BRANCH_REF" \
             .
       - name: Push dependabot-core-ci image to Docker registry
         if: env.DOCKER_LOGGED_IN == 'true'
         run: |
           docker push "$CORE_CI_IMAGE:latest"
-          docker push "$CORE_CI_IMAGE:ci--$BRANCH_REF"
+          docker push "$CORE_CI_IMAGE:branch--$BRANCH_REF"
       - name: Run ${{ matrix.suite.name }} tests
         run: |
-          docker run --env "CI=true" --env "DEPENDABOT_TEST_ACCESS_TOKEN=$DEPENDABOT_TEST_ACCESS_TOKEN" --env "SUITE_NAME=${{ matrix.suite.name }}" --rm "$CORE_CI_IMAGE" bash -c \
+          docker run \
+            -v "$(pwd)/.rubocop.yml:$CODE_DIR/.rubocop.yml" \
+            -v "$(pwd)/bin:$CODE_DIR/bin" \
+            -v "$(pwd)/common/.rubocop.yml:$CODE_DIR/common/.rubocop.yml" \
+            -v "$(pwd)/common/Gemfile:$CODE_DIR/common/Gemfile" \
+            -v "$(pwd)/common/dependabot-common.gemspec:$CODE_DIR/common/dependabot-common.gemspec" \
+            -v "$(pwd)/common/bin:$CODE_DIR/common/bin" \
+            -v "$(pwd)/common/lib:$CODE_DIR/common/lib" \
+            -v "$(pwd)/common/spec:$CODE_DIR/common/spec" \
+            -v "$(pwd)/common/script:$CODE_DIR/common/script" \
+            -v "$(pwd)/terraform/.rubocop.yml:$CODE_DIR/terraform/.rubocop.yml" \
+            -v "$(pwd)/terraform/Gemfile:$CODE_DIR/terraform/Gemfile" \
+            -v "$(pwd)/terraform/dependabot-terraform.gemspec:$CODE_DIR/terraform/dependabot-terraform.gemspec" \
+            -v "$(pwd)/terraform/lib:$CODE_DIR/terraform/lib" \
+            -v "$(pwd)/terraform/spec:$CODE_DIR/terraform/spec" \
+            -v "$(pwd)/terraform/script:$CODE_DIR/terraform/script" \
+            -v "$(pwd)/elm/.rubocop.yml:$CODE_DIR/elm/.rubocop.yml" \
+            -v "$(pwd)/elm/Gemfile:$CODE_DIR/elm/Gemfile" \
+            -v "$(pwd)/elm/dependabot-elm.gemspec:$CODE_DIR/elm/dependabot-elm.gemspec" \
+            -v "$(pwd)/elm/lib:$CODE_DIR/elm/lib" \
+            -v "$(pwd)/elm/spec:$CODE_DIR/elm/spec" \
+            -v "$(pwd)/elm/script:$CODE_DIR/elm/script" \
+            -v "$(pwd)/docker/.rubocop.yml:$CODE_DIR/docker/.rubocop.yml" \
+            -v "$(pwd)/docker/Gemfile:$CODE_DIR/docker/Gemfile" \
+            -v "$(pwd)/docker/dependabot-docker.gemspec:$CODE_DIR/docker/dependabot-docker.gemspec" \
+            -v "$(pwd)/docker/lib:$CODE_DIR/docker/lib" \
+            -v "$(pwd)/docker/spec:$CODE_DIR/docker/spec" \
+            -v "$(pwd)/docker/script:$CODE_DIR/docker/script" \
+            -v "$(pwd)/git_submodules/.rubocop.yml:$CODE_DIR/git_submodules/.rubocop.yml" \
+            -v "$(pwd)/git_submodules/Gemfile:$CODE_DIR/git_submodules/Gemfile" \
+            -v "$(pwd)/git_submodules/dependabot-git_submodules.gemspec:$CODE_DIR/git_submodules/dependabot-core.gemspec" \
+            -v "$(pwd)/git_submodules/lib:$CODE_DIR/git_submodules/lib" \
+            -v "$(pwd)/git_submodules/spec:$CODE_DIR/git_submodules/spec" \
+            -v "$(pwd)/git_submodules/script:$CODE_DIR/git_submodules/script" \
+            -v "$(pwd)/github_actions/.rubocop.yml:$CODE_DIR/github_actions/.rubocop.yml" \
+            -v "$(pwd)/github_actions/Gemfile:$CODE_DIR/github_actions/Gemfile" \
+            -v "$(pwd)/github_actions/dependabot-github_actions.gemspec:$CODE_DIR/github_actions/dependabot-core.gemspec" \
+            -v "$(pwd)/github_actions/lib:$CODE_DIR/github_actions/lib" \
+            -v "$(pwd)/github_actions/spec:$CODE_DIR/github_actions/spec" \
+            -v "$(pwd)/github_actions/script:$CODE_DIR/github_actions/script" \
+            -v "$(pwd)/python/.rubocop.yml:$CODE_DIR/python/.rubocop.yml" \
+            -v "$(pwd)/python/Gemfile:$CODE_DIR/python/Gemfile" \
+            -v "$(pwd)/python/dependabot-python.gemspec:$CODE_DIR/python/dependabot-python.gemspec" \
+            -v "$(pwd)/python/helpers:/opt/python" \
+            -v "$(pwd)/python/helpers:/opt/python/helpers" \
+            -v "$(pwd)/python/lib:$CODE_DIR/python/lib" \
+            -v "$(pwd)/python/spec:$CODE_DIR/python/spec" \
+            -v "$(pwd)/python/script:$CODE_DIR/python/script" \
+            -v "$(pwd)/nuget/.rubocop.yml:$CODE_DIR/nuget/.rubocop.yml" \
+            -v "$(pwd)/nuget/Gemfile:$CODE_DIR/nuget/Gemfile" \
+            -v "$(pwd)/nuget/dependabot-nuget.gemspec:$CODE_DIR/nuget/dependabot-core.gemspec" \
+            -v "$(pwd)/nuget/lib:$CODE_DIR/nuget/lib" \
+            -v "$(pwd)/nuget/spec:$CODE_DIR/nuget/spec" \
+            -v "$(pwd)/nuget/script:$CODE_DIR/nuget/script" \
+            -v "$(pwd)/maven/.rubocop.yml:$CODE_DIR/maven/.rubocop.yml" \
+            -v "$(pwd)/maven/Gemfile:$CODE_DIR/maven/Gemfile" \
+            -v "$(pwd)/maven/dependabot-maven.gemspec:$CODE_DIR/maven/dependabot-core.gemspec" \
+            -v "$(pwd)/maven/lib:$CODE_DIR/maven/lib" \
+            -v "$(pwd)/maven/spec:$CODE_DIR/maven/spec" \
+            -v "$(pwd)/maven/script:$CODE_DIR/maven/script" \
+            -v "$(pwd)/gradle/.rubocop.yml:$CODE_DIR/gradle/.rubocop.yml" \
+            -v "$(pwd)/gradle/Gemfile:$CODE_DIR/gradle/Gemfile" \
+            -v "$(pwd)/gradle/dependabot-gradle.gemspec:$CODE_DIR/gradle/dependabot-gradle.gemspec" \
+            -v "$(pwd)/gradle/lib:$CODE_DIR/gradle/lib" \
+            -v "$(pwd)/gradle/spec:$CODE_DIR/gradle/spec" \
+            -v "$(pwd)/gradle/script:$CODE_DIR/gradle/script" \
+            -v "$(pwd)/hex/.rubocop.yml:$CODE_DIR/hex/.rubocop.yml" \
+            -v "$(pwd)/hex/Gemfile:$CODE_DIR/hex/Gemfile" \
+            -v "$(pwd)/hex/dependabot-hex.gemspec:$CODE_DIR/hex/dependabot-hex.gemspec" \
+            -v "$(pwd)/hex/lib:$CODE_DIR/hex/lib" \
+            -v "$(pwd)/hex/spec:$CODE_DIR/hex/spec" \
+            -v "$(pwd)/hex/script:$CODE_DIR/hex/script" \
+            -v "$(pwd)/cargo/.rubocop.yml:$CODE_DIR/cargo/.rubocop.yml" \
+            -v "$(pwd)/cargo/Gemfile:$CODE_DIR/cargo/Gemfile" \
+            -v "$(pwd)/cargo/dependabot-cargo.gemspec:$CODE_DIR/cargo/dependabot-core.gemspec" \
+            -v "$(pwd)/cargo/lib:$CODE_DIR/cargo/lib" \
+            -v "$(pwd)/cargo/spec:$CODE_DIR/cargo/spec" \
+            -v "$(pwd)/cargo/script:$CODE_DIR/cargo/script" \
+            -v "$(pwd)/dep/.rubocop.yml:$CODE_DIR/dep/.rubocop.yml" \
+            -v "$(pwd)/dep/Gemfile:$CODE_DIR/dep/Gemfile" \
+            -v "$(pwd)/dep/dependabot-dep.gemspec:$CODE_DIR/dep/dependabot-dep.gemspec" \
+            -v "$(pwd)/dep/lib:$CODE_DIR/dep/lib" \
+            -v "$(pwd)/dep/spec:$CODE_DIR/dep/spec" \
+            -v "$(pwd)/dep/script:$CODE_DIR/dep/script" \
+            -v "$(pwd)/go_modules/.rubocop.yml:$CODE_DIR/go_modules/.rubocop.yml" \
+            -v "$(pwd)/go_modules/Gemfile:$CODE_DIR/go_modules/Gemfile" \
+            -v "$(pwd)/go_modules/dependabot-go_modules.gemspec:$CODE_DIR/go_modules/dependabot-go_modules.gemspec" \
+            -v "$(pwd)/go_modules/lib:$CODE_DIR/go_modules/lib" \
+            -v "$(pwd)/go_modules/spec:$CODE_DIR/go_modules/spec" \
+            -v "$(pwd)/go_modules/script:$CODE_DIR/go_modules/script" \
+            -v "$(pwd)/npm_and_yarn/.rubocop.yml:$CODE_DIR/npm_and_yarn/.rubocop.yml" \
+            -v "$(pwd)/npm_and_yarn/Gemfile:$CODE_DIR/npm_and_yarn/Gemfile" \
+            -v "$(pwd)/npm_and_yarn/dependabot-npm_and_yarn.gemspec:$CODE_DIR/npm_and_yarn/dependabot-npm_and_yarn.gemspec" \
+            -v "$(pwd)/npm_and_yarn/lib:$CODE_DIR/npm_and_yarn/lib" \
+            -v "$(pwd)/npm_and_yarn/spec:$CODE_DIR/npm_and_yarn/spec" \
+            -v "$(pwd)/npm_and_yarn/script:$CODE_DIR/npm_and_yarn/script" \
+            -v "$(pwd)/composer/.rubocop.yml:$CODE_DIR/composer/.rubocop.yml" \
+            -v "$(pwd)/composer/Gemfile:$CODE_DIR/composer/Gemfile" \
+            -v "$(pwd)/composer/dependabot-composer.gemspec:$CODE_DIR/composer/dependabot-composer.gemspec" \
+            -v "$(pwd)/composer/lib:$CODE_DIR/composer/lib" \
+            -v "$(pwd)/composer/spec:$CODE_DIR/composer/spec" \
+            -v "$(pwd)/composer/script:$CODE_DIR/composer/script" \
+            -v "$(pwd)/bundler/.rubocop.yml:$CODE_DIR/bundler/.rubocop.yml" \
+            -v "$(pwd)/bundler/Gemfile:$CODE_DIR/bundler/Gemfile" \
+            -v "$(pwd)/bundler/dependabot-bundler.gemspec:$CODE_DIR/bundler/dependabot-bundler.gemspec" \
+            -v "$(pwd)/bundler/lib:$CODE_DIR/bundler/lib" \
+            -v "$(pwd)/bundler/helpers:$CODE_DIR/bundler/helpers" \
+            -v "$(pwd)/bundler/spec:$CODE_DIR/bundler/spec" \
+            -v "$(pwd)/bundler/helpers:/opt/bundler" \
+            -v "$(pwd)/bundler/helpers:/opt/bundler/helpers" \
+            -v "$(pwd)/bundler/script:$CODE_DIR/bundler/script" \
+            -v "$(pwd)/omnibus/.rubocop.yml:$CODE_DIR/omnibus/.rubocop.yml" \
+            -v "$(pwd)/omnibus/Gemfile:$CODE_DIR/omnibus/Gemfile" \
+            -v "$(pwd)/omnibus/dependabot-omnibus.gemspec:$CODE_DIR/omnibus/dependabot-omnibus.gemspec" \
+            -v "$(pwd)/omnibus/lib:$CODE_DIR/omnibus/lib" \
+            -v "$(pwd)/omnibus/spec:$CODE_DIR/omnibus/spec" \
+            -v "$(pwd)/omnibus/script:$CODE_DIR/omnibus/script" \
+            --env "CI=true" \
+            --env "DEPENDABOT_TEST_ACCESS_TOKEN=$DEPENDABOT_TEST_ACCESS_TOKEN" \
+            --env "SUITE_NAME=${{ matrix.suite.name }}" \
+            --env "BUNDLE_PATH=/home/dependabot/.bundle" \
+            -w "$CODE_DIR" \
+            --rm "$CORE_CI_IMAGE" bash -c \
             "cd /home/dependabot/dependabot-core/${{ matrix.suite.path }} && ./script/ci-test"

.github/workflows/docker.yml

@@ -13,27 +13,33 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
+      - name: Prepare environment variables
+        run: |
+          echo "BASE_IMAGE=ubuntu:18.04" >> $GITHUB_ENV
+          echo "CORE_IMAGE=dependabot/dependabot-core" >> $GITHUB_ENV
       - name: Pull Docker base image & warm Docker cache
         run: |
-          docker pull "dependabot/dependabot-core:latest"
+          docker pull "$BASE_IMAGE"
+          docker pull "$CORE_IMAGE:latest"
       - name: Build dependabot-core image
         env:
           DOCKER_BUILDKIT: 1
         run: |
           docker build \
-            -t "dependabot/dependabot-core:latest" \
+            -t "$CORE_IMAGE:latest" \
             --build-arg BUILDKIT_INLINE_CACHE=1 \
+            --cache-from "$BASE_IMAGE" \
+            --cache-from "$CORE_IMAGE:latest" \
             .
       - name: Log in to the Docker registry
         run: |
           echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
-      - name: Push image to packages (latest)
-        if: "!contains(github.ref, 'refs/tags')"
+      - name: Push latest image
         run: |
-          docker push "dependabot/dependabot-core:latest"
-      - name: Push image to packages (tagged)
+          docker push "$CORE_IMAGE:latest"
+      - name: Push tagged image
         if: "contains(github.ref, 'refs/tags')"
         run: |
           VERSION="$(grep -Eo "[0-9]+\.[0-9]+\.[0-9]+" common/lib/dependabot/version.rb)"
-          docker tag "dependabot/dependabot-core:latest" "dependabot/dependabot-core:$VERSION"
-          docker push "dependabot/dependabot-core:$VERSION"
+          docker tag "$CORE_IMAGE:latest" "$CORE_IMAGE:$VERSION"
+          docker push "$CORE_IMAGE:$VERSION"

Dockerfile.ci

@@ -1,74 +1,7 @@
 FROM dependabot/dependabot-core
-ARG CODE_DIR=/home/dependabot/dependabot-core
-WORKDIR ${CODE_DIR}
-
-ENV BUNDLE_PATH="/home/dependabot/.bundle" \
-  BUNDLE_BIN=".bundle/binstubs" \
-  PATH=".bundle/binstubs:$PATH:/home/dependabot/.bundle/bin"
 
 RUN pyenv install -s 3.6.13
 RUN pyenv install -s 3.7.1
 RUN pyenv install -s 3.7.10
 
-COPY .rubocop.yml /home/dependabot/dependabot-core/
-
-RUN mkdir -p \
-  ${CODE_DIR}/bundler \
-  ${CODE_DIR}/cargo \
-  ${CODE_DIR}/common \
-  ${CODE_DIR}/composer \
-  ${CODE_DIR}/dep \
-  ${CODE_DIR}/docker \
-  ${CODE_DIR}/elm \
-  ${CODE_DIR}/git_submodules \
-  ${CODE_DIR}/github_actions \
-  ${CODE_DIR}/go_modules \
-  ${CODE_DIR}/gradle \
-  ${CODE_DIR}/hex \
-  ${CODE_DIR}/maven \
-  ${CODE_DIR}/npm_and_yarn \
-  ${CODE_DIR}/nuget \
-  ${CODE_DIR}/omnibus \
-  ${CODE_DIR}/python \
-  ${CODE_DIR}/terraform
-
-COPY common/lib/dependabot/version.rb ${CODE_DIR}/common/lib/dependabot/version.rb
-COPY common/Gemfile common/dependabot-common.gemspec ${CODE_DIR}/common/
-COPY bundler/Gemfile bundler/dependabot-bundler.gemspec ${CODE_DIR}/bundler/
-COPY cargo/Gemfile cargo/dependabot-cargo.gemspec ${CODE_DIR}/cargo/
-COPY composer/Gemfile composer/dependabot-composer.gemspec ${CODE_DIR}/composer/
-COPY dep/Gemfile dep/dependabot-dep.gemspec ${CODE_DIR}/dep/
-COPY docker/Gemfile docker/dependabot-docker.gemspec ${CODE_DIR}/docker/
-COPY elm/Gemfile elm/dependabot-elm.gemspec ${CODE_DIR}/elm/
-COPY git_submodules/Gemfile git_submodules/dependabot-git_submodules.gemspec ${CODE_DIR}/git_submodules/
-COPY github_actions/Gemfile github_actions/dependabot-github_actions.gemspec ${CODE_DIR}/github_actions/
-COPY go_modules/Gemfile go_modules/dependabot-go_modules.gemspec ${CODE_DIR}/go_modules/
-COPY gradle/Gemfile gradle/dependabot-gradle.gemspec ${CODE_DIR}/gradle/
-COPY hex/Gemfile hex/dependabot-hex.gemspec ${CODE_DIR}/hex/
-COPY maven/Gemfile maven/dependabot-maven.gemspec ${CODE_DIR}/maven/
-COPY npm_and_yarn/Gemfile npm_and_yarn/dependabot-npm_and_yarn.gemspec ${CODE_DIR}/npm_and_yarn/
-COPY nuget/Gemfile nuget/dependabot-nuget.gemspec ${CODE_DIR}/nuget/
-COPY python/Gemfile python/dependabot-python.gemspec ${CODE_DIR}/python/
-COPY terraform/Gemfile terraform/dependabot-terraform.gemspec ${CODE_DIR}/terraform/
-COPY omnibus/Gemfile omnibus/dependabot-omnibus.gemspec ${CODE_DIR}/omnibus/
-
-COPY common/ ${CODE_DIR}/common/
-COPY bundler/ ${CODE_DIR}/bundler/
-COPY cargo/ ${CODE_DIR}/cargo/
-COPY composer/ ${CODE_DIR}/composer/
-COPY dep/ ${CODE_DIR}/dep/
-COPY docker/ ${CODE_DIR}/docker/
-COPY elm/ ${CODE_DIR}/elm/
-COPY git_submodules/ ${CODE_DIR}/git_submodules/
-COPY github_actions/ ${CODE_DIR}/github_actions/
-COPY go_modules/ ${CODE_DIR}/go_modules/
-COPY gradle/ ${CODE_DIR}/gradle/
-COPY hex/ ${CODE_DIR}/hex/
-COPY maven/ ${CODE_DIR}/maven/
-COPY npm_and_yarn/ ${CODE_DIR}/npm_and_yarn/
-COPY nuget/ ${CODE_DIR}/nuget/
-COPY python/ ${CODE_DIR}/python/
-COPY terraform/ ${CODE_DIR}/terraform/
-COPY omnibus/ ${CODE_DIR}/omnibus/
-
 RUN git config --global user.name dependabot-ci && git config --global user.email [email protected]