Merge pull request #5628 from elasticspoon/parallel-tests

parallel_test gem experimentation

.github/workflows/ci.yml

@@ -0,0 +1,30 @@
+name: Continuous Integration
+
+on:
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - "doc/**"
+      - "**/*.md"
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  rspec:
+    strategy:
+      fail-fast: false
+      matrix:
+        groups: ["[0, 1, 2, 3]", "[4, 5, 6, 7]", "[8, 9, 10, 11]"]
+    uses: ./.github/workflows/rspec.yml
+    secrets: inherit
+    with:
+      groups: ${{ matrix.groups }}
+      group_count: 12 # the total number of test groups, must match the groups listed in the matrix.groups
+      parallel_processes_count: 4 # the number of parallel processes to run tests in worker, must match the size of the
+      # inner arrays in the matrix.groups
+  combine_and_report:
+    uses: ./.github/workflows/combine_and_report.yml
+    needs: [rspec]
+    secrets: inherit

.github/workflows/combine_and_report.yml

@@ -0,0 +1,47 @@
+on:
+  workflow_call:
+jobs:
+  combine_and_report:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Project
+        uses: actions/checkout@v4
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: artifacts
+      - name: Decompress chunk test reports
+        run: |
+          find artifacts -name "test_reports*.zip" -exec unzip -d test_reports {} \;
+          find test_reports -name "**/test_reports*.zip" -exec unzip -d test_reports {} \;
+      - name: Merge parallel runtime log parts
+        if: github.repository == github.event.pull_request.head.repo.full_name
+        run: |
+          cat artifacts/**/parallel_runtime_rspec*.log > parallel_runtime.log
+      - name: Upload log file to Azure Blob Storage
+        if: github.repository == github.event.pull_request.head.repo.full_name
+        env:
+          AZURE_STORAGE_KEY: ${{ secrets.STORAGE_ACCESS_KEY }}
+          AZURE_STORAGE_ACCOUNT: ${{ secrets.ACCOUNT_NAME }}
+          STORAGE_CONTAINER: ${{ secrets.STORAGE_CONTAINER }}
+        run: |
+          az storage blob upload \
+          -c $STORAGE_CONTAINER \
+          --file parallel_runtime.log \
+          -n parallel_runtime.log \
+          --overwrite true
+
+      - name: Test Summary
+        id: test_summary
+        uses: test-summary/action@v2
+        with:
+          paths: |
+            test_reports/**/rspec*.xml
+      - name: Set job status
+        #         In this step we set the status of the job. Normally in case of failures, the next steps fail, so we have to
+        #         use `if: always()` to make sure the next steps run.
+        if: ${{ steps.test_summary.outputs.failed > 0 }}
+        uses: actions/github-script@v6
+        with:
+          script: |
+            core.setFailed('There are test failures')

.github/workflows/docker.yml

@@ -13,6 +13,9 @@ on:
     paths-ignore:
       - "doc/**"
       - "**/*.md"
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
 jobs:
   docker:

.github/workflows/rspec.yml

@@ -1,22 +1,31 @@
 name: rspec
 
 on:
-  push:
-    branches:
-      - main
-    paths-ignore:
-      - "doc/**"
-      - "**/*.md"
-  pull_request:
-    branches:
-      - main
+  workflow_call:
+    inputs:
+      groups:
+        required: true
+        type: string
+      group_count:
+        required: true
+        type: number
+      parallel_processes_count:
+        required: true
+        type: number
+env:
+  GROUPS_COMMA: ${{ join(fromJSON(inputs.groups), ',') }}
+  GROUPS_UNDERSCORE: ${{ join(fromJSON(inputs.groups), '_') }}
 
 jobs:
   rspec:
+    name: RSpec Groups ${{ inputs.groups }}
     runs-on: ubuntu-latest
+    timeout-minutes: 10
     env:
       RAILS_ENV: test
-
+      BUNDLE_WITHOUT: "development"
+      CI_TOTAL_JOBS: ${{ inputs.group_count }}
+      CI_JOB_INDEX: ${{ inputs.groups }}
     services:
       db:
         image: postgres:14.8
@@ -29,21 +38,42 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
-
     steps:
       - uses: actions/checkout@v4
 
+      - name: Download parallel runtime log from Azure Blob Storage
+        if: github.repository == github.event.pull_request.head.repo.full_name
+        env:
+          AZURE_STORAGE_KEY: ${{ secrets.STORAGE_ACCESS_KEY }}
+          AZURE_STORAGE_ACCOUNT: ${{ secrets.ACCOUNT_NAME }}
+          STORAGE_CONTAINER: ${{ secrets.STORAGE_CONTAINER }}
+        run: |
+          echo $STORAGE_CONTAINER
+          echo ${{ secrets.STORAGE_CONTAINER }}
+          az storage blob download \
+          -c $STORAGE_CONTAINER \
+          --file old_parallel_runtime.log \
+          -n parallel_runtime.log
+
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
           bundler-cache: true
 
+      - name: Set up JS
+        uses: actions/setup-node@v4
+        with:
+          node-version: lts/gallium
+          cache: 'yarn'
+      - run: yarn
+
       - name: Install PostgreSQL client
         run: |
           sudo apt-get -yqq install libpq-dev
 
-      - name: Build App
+      - name: Setup Parallel Database
         env:
+          RAILS_ENV: test
           POSTGRES_HOST: localhost
           DATABASE_HOST: localhost
           POSTGRES_USER: postgres
@@ -52,32 +82,68 @@ jobs:
           POSTGRES_HOST_AUTH_METHOD: trust
           POSTGRES_PORT: 5432
         run: |
-          yarn
-          bundle exec rake db:create
-          bundle exec rake db:schema:load
+          echo "setting up database"
+          bundle exec rake parallel:create
+          bundle exec rake parallel:rake[db:schema:load]
+          echo "done"
+
+      - name: Build App
+        run: |
           bundle exec rails css:build
           bundle exec rails javascript:build
 
-      - name: Run rspec and upload code coverage
+      - name: Run rspec group ${{ inputs.group }}
         env:
+          RAILS_ENV: test
+          POSTGRES_HOST: localhost
           DATABASE_HOST: localhost
           POSTGRES_USER: postgres
           CASA_DATABASE_PASSWORD: password
+          POSTGRES_PASSWORD: password
           POSTGRES_HOST_AUTH_METHOD: trust
+          POSTGRES_PORT: 5432
           RUN_SIMPLECOV: true
           CC_TEST_REPORTER_ID: 31464536e34ab26588cb951d0fa6b5898abdf401dbe912fd47274df298e432ac
+        continue-on-error: true
         run: |
           curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
           chmod +x ./cc-test-reporter
           ./cc-test-reporter before-build
-          RUBYOPT='-W:no-deprecated -W:no-experimental' bundle exec rspec
+
+          RUBYOPT='-W:no-deprecated -W:no-experimental' bundle exec parallel_rspec \
+          -n "${CI_TOTAL_JOBS}" \
+          --only-group "${CI_JOB_INDEX}" \
+          --runtime-log old_parallel_runtime.log \
+          --verbose-command ./spec
+
           ./cc-test-reporter after-build --exit-code $?
+          cat tmp/spec_summary.log
+
+      - name: Compress reports
+        run: |
+          zip -r test_reports_${{ env.GROUPS_UNDERSCORE }}.zip tmp/reports
+
+      - name: Compress log
+        if: github.repository == github.event.pull_request.head.repo.full_name
+        run: |
+          mv tmp/parallel_runtime.log parallel_runtime_rspec_${{ env.GROUPS_UNDERSCORE }}.log
+
+      - name: Upload test reports
+        uses: actions/upload-artifact@v4
+        with:
+          name: test_reports_${{ env.GROUPS_UNDERSCORE }}.zip
+          path: test_reports_${{ env.GROUPS_UNDERSCORE }}.zip
+
+      - name: Upload file parallel tests runtime log
+        if: github.repository == github.event.pull_request.head.repo.full_name
+        uses: actions/upload-artifact@v4
+        with:
+          name: parallel_runtime_rspec_${{ env.GROUPS_UNDERSCORE }}.log
+          path: parallel_runtime_rspec_${{ env.GROUPS_UNDERSCORE }}.log
 
-      - name: Archive selenium screenshots
-        if: ${{ failure() }}
+      - name: Upload Selenium Screenshots
         uses: actions/upload-artifact@v4
         with:
-          name: selenium-screenshots
-          path: |
-            ${{ github.workspace }}/tmp/capybara/*.png
-            ${{ github.workspace }}/tmp/capybara/*.html
+          name: screenshots_${{ env.GROUPS_UNDERSCORE }}
+          path: ${{ github.workspace }}/tmp/screenshots${{ env.GROUPS_UNDERSCORE }}/
+          if-no-files-found: ignore

.rspec_parallel

@@ -0,0 +1,5 @@
+--format RspecJunitFormatter
+--out tmp/reports/rspec_<%= ENV["GROUPS_UNDERSCORE"] %>_<%= ENV["TEST_ENV_NUMBER"] %>.xml
+--format ParallelTests::RSpec::RuntimeLogger --out tmp/parallel_runtime.log
+--format ParallelTests::RSpec::SummaryLogger --out tmp/spec_summary.log
+--format ParallelTests::RSpec::FailuresLogger --out tmp/failing_specs.log

Gemfile

@@ -66,6 +66,8 @@ group :development, :test do
   gem "rswag-specs"
   gem "shoulda-matchers"
   gem "standard", "~> 1.35.1"
+  gem "parallel_tests"
+  gem "rspec_junit_formatter"
 end
 
 group :development do

Gemfile.lock

@@ -339,6 +339,8 @@ GEM
       bigdecimal (>= 3.0)
     orm_adapter (0.5.0)
     parallel (1.24.0)
+    parallel_tests (4.7.1)
+      parallel
     paranoia (2.6.3)
       activerecord (>= 5.1, < 7.2)
     parser (3.3.1.0)
@@ -440,6 +442,8 @@ GEM
       rspec-mocks (~> 3.12)
       rspec-support (~> 3.12)
     rspec-support (3.12.1)
+    rspec_junit_formatter (0.6.0)
+      rspec-core (>= 2, < 4, != 2.12.0)
     rswag-api (2.13.0)
       activesupport (>= 3.1, < 7.2)
       railties (>= 3.1, < 7.2)
@@ -561,15 +565,10 @@ GEM
     zeitwerk (2.6.13)
 
 PLATFORMS
-  arm64-darwin-20
   arm64-darwin-21
-  arm64-darwin-22
   ruby
-  x86_64-darwin-18
   x86_64-darwin-19
-  x86_64-darwin-20
   x86_64-darwin-21
-  x86_64-darwin-22
   x86_64-linux
 
 DEPENDENCIES
@@ -614,6 +613,7 @@ DEPENDENCIES
   net-smtp
   noticed
   oj
+  parallel_tests
   paranoia
   pdf-forms
   pg
@@ -630,6 +630,7 @@ DEPENDENCIES
   request_store
   rexml
   rspec-rails
+  rspec_junit_formatter
   rswag-api
   rswag-specs
   rswag-ui

config/database.yml

@@ -13,7 +13,7 @@ development:
 
 test:
   <<: *default
-  database: casa_test
+  database: casa_test<%= ENV["TEST_ENV_NUMBER"] %>
   host: <%= ENV.fetch("DATABASE_HOST") { } %>
 
 production:

config/storage.yml

@@ -1,6 +1,6 @@
 test:
   service: Disk
-  root: <%= Rails.root.join("tmp/storage") %>
+  root: <%= Rails.root.join("tmp/storage#{ENV['TEST_ENV_NUMBER']}") %>
 
 local:
   service: Disk

spec/lib/tasks/emancipation_checklist_reminder_spec.rb

@@ -2,7 +2,7 @@
 require "rake"
 Rake.application.rake_require "tasks/emancipation_checklist_reminder"
 
-RSpec.describe "lib/tasks/emancipation_checklist_reminder.rake" do
+RSpec.describe "lib/tasks/emancipation_checklist_reminder.rake", ci_only: true do
   let(:rake_task) { Rake::Task["emancipation_checklist_reminder"].invoke }
 
   before do

spec/lib/tasks/volunteer_birthday_reminder_spec.rb

@@ -2,7 +2,7 @@
 require "rake"
 Rake.application.rake_require "tasks/volunteer_birthday_reminder"
 
-RSpec.describe "lib/tasks/volunteer_birthday_reminder.rake" do
+RSpec.describe "lib/tasks/volunteer_birthday_reminder.rake", ci_only: true do
   let(:rake_task) { Rake::Task["volunteer_birthday_reminder"].invoke }
   let(:now) { Date.new(2022, 10, 15) }
   let(:this_month) { now.month }

spec/rails_helper.rb

@@ -76,4 +76,13 @@
   def pre_transition_aged_youth_age
     Date.current - CasaCase::TRANSITION_AGE.years
   end
+
+  config.around do |example|
+    Capybara.server_port = 7654 + ENV["TEST_ENV_NUMBER"].to_i
+    example.run
+  end
+
+  unless ENV["GITHUB_ACTIONS"]
+    config.filter_run_excluding :ci_only
+  end
 end

spec/spec_helper.rb

@@ -6,6 +6,8 @@
 if ENV["RUN_SIMPLECOV"]
   require "simplecov"
   SimpleCov.start do
+    command_name "Job #{ENV["TEST_ENV_NUMBER"]}" if ENV["TEST_ENV_NUMBER"]
+
     add_filter "/spec/"
     add_filter "/lib/tasks/auto_annotate_models.rake"
     add_group "Models", "/app/models"

spec/support/capybara.rb

@@ -14,6 +14,16 @@
 # disable CSS transitions and js animations
 Capybara.disable_animation = true
 
+Capybara::Screenshot.autosave_on_failure = true
+
+module Capybara
+  module Screenshot
+    def self.capybara_tmp_path
+      Rails.root.join("tmp", "screenshots#{ENV["GROUPS_UNDERSCORE"]}")
+    end
+  end
+end
+
 options = Selenium::WebDriver::Chrome::Options.new
 options.add_argument("--disable-gpu")
 options.add_argument("--ignore-certificate-errors")