From 0877a99e035694a01a20adad62814f978e2107b5 Mon Sep 17 00:00:00 2001 From: Jurre Stender Date: Thu, 1 Apr 2021 12:46:15 +0200 Subject: [PATCH 1/2] CI: Only bundle install for ecosystem under test Currently in the CI container we bundle install for _every_ package, this ends up taking 4 to 5 minutes on every build for every ecosystem. Since the tests are split up per ecosystem, we only need to install the gems for the package that is being tested in this matrix step. This also introduces a `script/ci-test` per ecosystem, it currently runs both rubocop and rspec for all ecosystems, but the expectation is that these scripts might diverge per ecosystem. Running both rspec and rubocop in the same script is done to prevent us from having to `bundle install` for each of those steps. We could take this approach a step further and split out a CI container per ecosystem, and that would let us run rubocop and rspec as separate steps, but I don't think this is currently worth it. This should shave about 5 minutes of every matrix build. --- .github/workflows/ci.yml | 5 +---- Dockerfile.ci | 19 ------------------- bundler/script/ci-test | 5 +++++ cargo/script/ci-test | 5 +++++ common/script/ci-test | 5 +++++ composer/script/ci-test | 5 +++++ dep/script/ci-test | 5 +++++ docker/script/ci-test | 5 +++++ elm/script/ci-test | 5 +++++ git_submodules/script/ci-test | 5 +++++ github_actions/script/ci-test | 5 +++++ go_modules/script/ci-test | 5 +++++ gradle/script/ci-test | 5 +++++ hex/script/ci-test | 5 +++++ maven/script/ci-test | 5 +++++ npm_and_yarn/script/ci-test | 5 +++++ nuget/script/ci-test | 5 +++++ omnibus/script/ci-test | 5 +++++ python/script/ci-test | 5 +++++ terraform/script/ci-test | 5 +++++ 20 files changed, 91 insertions(+), 23 deletions(-) create mode 100755 bundler/script/ci-test create mode 100755 cargo/script/ci-test create mode 100755 common/script/ci-test create mode 100755 composer/script/ci-test create mode 100755 dep/script/ci-test create mode 100755 docker/script/ci-test create mode 100755 elm/script/ci-test create mode 100755 git_submodules/script/ci-test create mode 100755 github_actions/script/ci-test create mode 100755 go_modules/script/ci-test create mode 100755 gradle/script/ci-test create mode 100755 hex/script/ci-test create mode 100755 maven/script/ci-test create mode 100755 npm_and_yarn/script/ci-test create mode 100755 nuget/script/ci-test create mode 100755 omnibus/script/ci-test create mode 100755 python/script/ci-test create mode 100755 terraform/script/ci-test diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 009af81fb0..3885cfe603 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,9 +102,6 @@ jobs: if: matrix.suite.name == 'python' run: | docker run --rm "$CORE_CI_IMAGE" bash -c "pyenv exec flake8 python/helpers/. --count --exclude=./.*,./python/spec/fixtures --show-source --statistics" - - name: Run Ruby Rubocop linting - run: | - docker run --rm "$CORE_CI_IMAGE" bash -c "cd /home/dependabot/dependabot-core/${{ matrix.suite.path }} && bundle exec rubocop ." - name: Run js linting and tests if: matrix.suite.name == 'npm_and_yarn' run: | @@ -123,4 +120,4 @@ jobs: - name: Run ${{ matrix.suite.name }} tests with rspec run: | docker run --env "CI=true" --env "DEPENDABOT_TEST_ACCESS_TOKEN=$DEPENDABOT_TEST_ACCESS_TOKEN" --env "SUITE_NAME=${{ matrix.suite.name }}" --rm "$CORE_CI_IMAGE" bash -c \ - "cd /home/dependabot/dependabot-core/${{ matrix.suite.path }} && bundle exec rspec spec" + "cd /home/dependabot/dependabot-core/${{ matrix.suite.path }} && ./script/ci-test" diff --git a/Dockerfile.ci b/Dockerfile.ci index b5a40bf986..eb5242c9a4 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -48,25 +48,6 @@ COPY python/Gemfile python/dependabot-python.gemspec ${CODE_DIR}/python/ COPY terraform/Gemfile terraform/dependabot-terraform.gemspec ${CODE_DIR}/terraform/ COPY omnibus/Gemfile omnibus/dependabot-omnibus.gemspec ${CODE_DIR}/omnibus/ -RUN cd common && bundle install -RUN cd bundler && bundle install -RUN cd cargo && bundle install -RUN cd composer && bundle install -RUN cd dep && bundle install -RUN cd docker && bundle install -RUN cd elm && bundle install -RUN cd git_submodules && bundle install -RUN cd github_actions && bundle install -RUN cd go_modules && bundle install -RUN cd gradle && bundle install -RUN cd hex && bundle install -RUN cd maven && bundle install -RUN cd npm_and_yarn && bundle install -RUN cd nuget && bundle install -RUN cd python && bundle install -RUN cd terraform && bundle install -RUN cd omnibus && bundle install - COPY common/ ${CODE_DIR}/common/ COPY bundler/ ${CODE_DIR}/bundler/ COPY cargo/ ${CODE_DIR}/cargo/ diff --git a/bundler/script/ci-test b/bundler/script/ci-test new file mode 100755 index 0000000000..64621ed561 --- /dev/null +++ b/bundler/script/ci-test @@ -0,0 +1,5 @@ +#!/bin/sh + +bundle install +bundle exec rubocop . +bundle exec rspec spec diff --git a/cargo/script/ci-test b/cargo/script/ci-test new file mode 100755 index 0000000000..64621ed561 --- /dev/null +++ b/cargo/script/ci-test @@ -0,0 +1,5 @@ +#!/bin/sh + +bundle install +bundle exec rubocop . +bundle exec rspec spec diff --git a/common/script/ci-test b/common/script/ci-test new file mode 100755 index 0000000000..64621ed561 --- /dev/null +++ b/common/script/ci-test @@ -0,0 +1,5 @@ +#!/bin/sh + +bundle install +bundle exec rubocop . +bundle exec rspec spec diff --git a/composer/script/ci-test b/composer/script/ci-test new file mode 100755 index 0000000000..64621ed561 --- /dev/null +++ b/composer/script/ci-test @@ -0,0 +1,5 @@ +#!/bin/sh + +bundle install +bundle exec rubocop . +bundle exec rspec spec diff --git a/dep/script/ci-test b/dep/script/ci-test new file mode 100755 index 0000000000..64621ed561 --- /dev/null +++ b/dep/script/ci-test @@ -0,0 +1,5 @@ +#!/bin/sh + +bundle install +bundle exec rubocop . +bundle exec rspec spec diff --git a/docker/script/ci-test b/docker/script/ci-test new file mode 100755 index 0000000000..64621ed561 --- /dev/null +++ b/docker/script/ci-test @@ -0,0 +1,5 @@ +#!/bin/sh + +bundle install +bundle exec rubocop . +bundle exec rspec spec diff --git a/elm/script/ci-test b/elm/script/ci-test new file mode 100755 index 0000000000..64621ed561 --- /dev/null +++ b/elm/script/ci-test @@ -0,0 +1,5 @@ +#!/bin/sh + +bundle install +bundle exec rubocop . +bundle exec rspec spec diff --git a/git_submodules/script/ci-test b/git_submodules/script/ci-test new file mode 100755 index 0000000000..64621ed561 --- /dev/null +++ b/git_submodules/script/ci-test @@ -0,0 +1,5 @@ +#!/bin/sh + +bundle install +bundle exec rubocop . +bundle exec rspec spec diff --git a/github_actions/script/ci-test b/github_actions/script/ci-test new file mode 100755 index 0000000000..64621ed561 --- /dev/null +++ b/github_actions/script/ci-test @@ -0,0 +1,5 @@ +#!/bin/sh + +bundle install +bundle exec rubocop . +bundle exec rspec spec diff --git a/go_modules/script/ci-test b/go_modules/script/ci-test new file mode 100755 index 0000000000..64621ed561 --- /dev/null +++ b/go_modules/script/ci-test @@ -0,0 +1,5 @@ +#!/bin/sh + +bundle install +bundle exec rubocop . +bundle exec rspec spec diff --git a/gradle/script/ci-test b/gradle/script/ci-test new file mode 100755 index 0000000000..64621ed561 --- /dev/null +++ b/gradle/script/ci-test @@ -0,0 +1,5 @@ +#!/bin/sh + +bundle install +bundle exec rubocop . +bundle exec rspec spec diff --git a/hex/script/ci-test b/hex/script/ci-test new file mode 100755 index 0000000000..64621ed561 --- /dev/null +++ b/hex/script/ci-test @@ -0,0 +1,5 @@ +#!/bin/sh + +bundle install +bundle exec rubocop . +bundle exec rspec spec diff --git a/maven/script/ci-test b/maven/script/ci-test new file mode 100755 index 0000000000..64621ed561 --- /dev/null +++ b/maven/script/ci-test @@ -0,0 +1,5 @@ +#!/bin/sh + +bundle install +bundle exec rubocop . +bundle exec rspec spec diff --git a/npm_and_yarn/script/ci-test b/npm_and_yarn/script/ci-test new file mode 100755 index 0000000000..64621ed561 --- /dev/null +++ b/npm_and_yarn/script/ci-test @@ -0,0 +1,5 @@ +#!/bin/sh + +bundle install +bundle exec rubocop . +bundle exec rspec spec diff --git a/nuget/script/ci-test b/nuget/script/ci-test new file mode 100755 index 0000000000..64621ed561 --- /dev/null +++ b/nuget/script/ci-test @@ -0,0 +1,5 @@ +#!/bin/sh + +bundle install +bundle exec rubocop . +bundle exec rspec spec diff --git a/omnibus/script/ci-test b/omnibus/script/ci-test new file mode 100755 index 0000000000..64621ed561 --- /dev/null +++ b/omnibus/script/ci-test @@ -0,0 +1,5 @@ +#!/bin/sh + +bundle install +bundle exec rubocop . +bundle exec rspec spec diff --git a/python/script/ci-test b/python/script/ci-test new file mode 100755 index 0000000000..64621ed561 --- /dev/null +++ b/python/script/ci-test @@ -0,0 +1,5 @@ +#!/bin/sh + +bundle install +bundle exec rubocop . +bundle exec rspec spec diff --git a/terraform/script/ci-test b/terraform/script/ci-test new file mode 100755 index 0000000000..64621ed561 --- /dev/null +++ b/terraform/script/ci-test @@ -0,0 +1,5 @@ +#!/bin/sh + +bundle install +bundle exec rubocop . +bundle exec rspec spec From e05853d37f5aedf6b72df198191ade9f69a87a49 Mon Sep 17 00:00:00 2001 From: Jurre Date: Thu, 1 Apr 2021 15:16:17 +0200 Subject: [PATCH 2/2] Update .github/workflows/ci.yml Co-authored-by: Philip Harrison --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3885cfe603..923639c194 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,7 +117,7 @@ jobs: run: | docker run --rm "$CORE_CI_IMAGE" bash -c \ "cd /home/dependabot/dependabot-core/bundler/helpers/v2 && BUNDLER_VERSION=2 bundle install && BUNDLER_VERSION=2 bundle exec rspec spec" - - name: Run ${{ matrix.suite.name }} tests with rspec + - name: Run ${{ matrix.suite.name }} tests run: | docker run --env "CI=true" --env "DEPENDABOT_TEST_ACCESS_TOKEN=$DEPENDABOT_TEST_ACCESS_TOKEN" --env "SUITE_NAME=${{ matrix.suite.name }}" --rm "$CORE_CI_IMAGE" bash -c \ "cd /home/dependabot/dependabot-core/${{ matrix.suite.path }} && ./script/ci-test"