diff --git a/.circleci/config.yml b/.circleci/config.yml index 189a9709d90..8e08bd15f98 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -65,7 +65,7 @@ save_bundle_checksum: &save_bundle_checksum command: | if [ "$CI_BUNDLE_CACHE_HIT" != 1 ]; then # Recompute gemfiles/*.lock checksum, as those files might have changed - cat Gemfile Gemfile.lock Appraisals gemfiles/*.gemfile.lock | md5sum > .circleci/bundle_checksum + cat Gemfile Gemfile.lock Appraisals gemfiles/*.gemfile gemfiles/*.gemfile.lock | md5sum > .circleci/bundle_checksum fi cp .circleci/bundle_checksum /usr/local/bundle/bundle_checksum step_bundle_install: &step_bundle_install @@ -82,16 +82,12 @@ step_appraisal_install: &step_appraisal_install name: Install Appraisal gems command: | if [ "$CI_BUNDLE_CACHE_HIT" != 1 ]; then - bundle exec appraisal install + bundle exec appraisal generate + bundle exec rake dependency:install else - bundle exec appraisal generate # Generate the appraisal files to match the lockfiles in the tree + bundle exec appraisal generate echo "All required gems were found in cache." fi -step_appraisal_update: &step_appraisal_update - run: - name: Update Appraisal gems - command: | # Remove all generated gemfiles and lockfiles, resolve, and install dependencies again - bundle exec appraisal update step_compute_bundle_checksum: &step_compute_bundle_checksum run: name: Compute bundle checksum @@ -100,7 +96,7 @@ step_compute_bundle_checksum: &step_compute_bundle_checksum # updating the gemset lock files produces extremely large commits. command: | bundle lock # Create Gemfile.lock - cat Gemfile Gemfile.lock Appraisals gemfiles/*.gemfile.lock | md5sum > .circleci/bundle_checksum + cat Gemfile Gemfile.lock Appraisals gemfiles/*.gemfile gemfiles/*.gemfile.lock | md5sum > .circleci/bundle_checksum step_run_all_tests: &step_run_all_tests run: name: Run tests @@ -164,6 +160,12 @@ orbs: <<: *test_job_default steps: - checkout + - run: + name: Know the environment + command: | + ruby -v + gem -v + bundle -v - restore_cache: keys: - '{{ .Environment.CIRCLE_CACHE_VERSION }}-bundled-repo-<>-{{ .Environment.CIRCLE_SHA1 }}' @@ -175,17 +177,7 @@ orbs: - bundle-{{ .Environment.CIRCLE_CACHE_VERSION }}-{{ checksum ".circleci/images/primary/binary_version" }}-<>-{{ checksum "lib/datadog/version.rb" }} - *check_exact_bundle_cache_hit - *step_bundle_install - - when: - condition: - equal: [ << parameters.edge >>, true ] - steps: - - *step_appraisal_update # Run on latest version of all gems we integrate with - - when: - condition: - not: - equal: [ << parameters.edge >>, true ] - steps: - - *step_appraisal_install # Run on a stable set of gems we integrate with + - *step_appraisal_install # Run on a stable set of gems we integrate with - *save_bundle_checksum - save_cache: key: '{{ .Environment.CIRCLE_CACHE_VERSION }}-bundled-repo-<>-{{ .Environment.CIRCLE_SHA1 }}' @@ -293,6 +285,10 @@ orbs: curl --silent --show-error --location --fail --retry 3 --output /usr/local/bin/dockerize $DOCKERIZE_URL chmod +x /usr/local/bin/dockerize dockerize --version + # Test agent uses `jq` to check results + - run: + name: Install jq + command: apt update && apt install jq -y # Wait for containers to start - docker-wait: port: 5432 @@ -315,10 +311,6 @@ orbs: host: "testagent" port: 9126 - *step_run_all_tests - - run: - # Test agent uses `jq` to check results - name: Install jq - command: apt update && apt install jq -y - *step_get_test_agent_trace_check_results - store_test_results: path: /tmp/rspec diff --git a/.circleci/images/primary/binary_version b/.circleci/images/primary/binary_version index 408b885ce54..bf7aeeb3e2e 100644 --- a/.circleci/images/primary/binary_version +++ b/.circleci/images/primary/binary_version @@ -1 +1 @@ -461 +462 diff --git a/Rakefile b/Rakefile index 08d21a2cfdc..a0a0a2fbd69 100644 --- a/Rakefile +++ b/Rakefile @@ -22,6 +22,7 @@ Dir.glob('tasks/*.rake').each { |r| import r } TEST_METADATA = eval(File.read('Matrixfile')).freeze # rubocop:disable Security/Eval +# rubocop:disable Metrics/BlockLength namespace :test do desc 'Run all tests' task all: TEST_METADATA.map { |k, _| "test:#{k}" } @@ -56,13 +57,13 @@ namespace :test do end candidates.each do |appraisal_group, _| - command = if appraisal_group.empty? - "bundle exec rake #{spec_task}" - else - gemfile = File.join(File.dirname(__FILE__), 'gemfiles', "#{ruby_runtime}-#{appraisal_group}.gemfile".tr('-', '_')) - "env BUNDLE_GEMFILE=#{gemfile} bundle exec rake #{spec_task}" - end - + env = if appraisal_group.empty? + {} + else + gemfile = File.join(File.dirname(__FILE__), 'gemfiles', "#{ruby_runtime}-#{appraisal_group}.gemfile".tr('-', '_')) + { 'BUNDLE_GEMFILE' => gemfile } + end + command = "bundle check || bundle install && bundle exec rake #{spec_task}" command += "'[#{spec_arguments}]'" if spec_arguments total_executors = ENV.key?('CIRCLE_NODE_TOTAL') ? ENV['CIRCLE_NODE_TOTAL'].to_i : nil @@ -71,9 +72,9 @@ namespace :test do if total_executors && current_executor && total_executors > 1 @execution_count ||= 0 @execution_count += 1 - sh(command) if @execution_count % total_executors == current_executor + Bundler.with_unbundled_env { sh(env, command) } if @execution_count % total_executors == current_executor else - sh(command) + Bundler.with_unbundled_env { sh(env, command) } end end end @@ -81,7 +82,6 @@ namespace :test do end desc 'Run RSpec' -# rubocop:disable Metrics/BlockLength namespace :spec do # REMINDER: If adding a new task here, make sure also add it to the `Matrixfile` task all: [:main, :benchmark, diff --git a/tasks/dependency.rake b/tasks/dependency.rake index 4af85eb1e9f..362baa67f28 100644 --- a/tasks/dependency.rake +++ b/tasks/dependency.rake @@ -1,49 +1,50 @@ -require 'open3' - require_relative 'appraisal_conversion' -task :dep => :dependency -task :dependency => %w[dependency:lock] namespace :dependency do - # rubocop:disable Style/MultilineBlockChain - Dir.glob(AppraisalConversion.gemfile_pattern).each do |gemfile| - # desc "Lock the dependencies for #{gemfile}" - task gemfile do + # Replacement for `bundle exec appraisal list` + desc "List dependencies for #{AppraisalConversion.runtime_identifier}" + task :list do |t, args| + pattern = args.extras.any? ? args.extras : AppraisalConversion.gemfile_pattern + + gemfiles = Dir.glob(pattern, base: AppraisalConversion.root_path) + + puts "Ahoy! Here is a list of gemfiles you are looking for:\n\n" + + puts "========================================\n" + puts gemfiles + puts "========================================\n" + + puts "You can do a bunch of cool stuff by assigning a gemfile path to the BUNDLE_GEMFILE environment variable, like:\n" + puts "`BUNDLE_GEMFILE=#{gemfiles.sample} bundle install`\n\n" + end + + # Replacement for `bundle exec appraisal bundle lock` + desc "Lock dependencies for #{AppraisalConversion.runtime_identifier}" + task :lock do |t, args| + pattern = args.extras.any? ? args.extras : AppraisalConversion.gemfile_pattern + + gemfiles = Dir.glob(pattern) + + gemfiles.each do |gemfile| Bundler.with_unbundled_env do command = +'bundle lock' command << ' --add-platform x86_64-linux aarch64-linux' unless RUBY_PLATFORM == 'java' - output, = Open3.capture2e({ 'BUNDLE_GEMFILE' => gemfile.to_s }, command) + sh({ 'BUNDLE_GEMFILE' => gemfile.to_s }, command) + end + end + end - puts output + # Replacement for `bundle exec appraisal install` + desc "Install dependencies for #{AppraisalConversion.runtime_identifier}" + task :install => :lock do |t, args| + pattern = args.extras.any? ? args.extras : AppraisalConversion.gemfile_pattern + + gemfiles = Dir.glob(pattern) + + gemfiles.each do |gemfile| + Bundler.with_unbundled_env do + sh({ 'BUNDLE_GEMFILE' => gemfile.to_s }, 'bundle check || bundle install') end end - end.tap do |gemfiles| - desc "Lock the dependencies for #{AppraisalConversion.runtime_identifier}" - # WHY can't we use `multitask :lock => gemfiles` here? - # - # Running bundler in parallel has various race conditions - # - # Race condition with the file system, particularly worse with JRuby. - # For instance, `Errno::ENOENT: No such file or directory - bundle` is raised with JRuby 9.2 - - # Even with CRuby, `simplcov` declaration with `github` in Gemfile causes - # race condition for the local gem cache with the following error: - - # ``` - # [/usr/local/bundle/bundler/gems/simplecov-3bb6b7ee58bf/simplecov.gemspec] isn't a Gem::Specification (NilClass instead). - # ``` - - # and - - # ``` - # fatal: Unable to create '/usr/local/bundle/bundler/gems/simplecov-3bb6b7ee58bf/.git/index.lock': File exists. - # Another git process seems to be running in this repository, e.g. - # an editor opened by 'git commit'. Please make sure all processes - # are terminated then try again. If it still fails, a git process - # may have crashed in this repository earlier: - # remove the file manually to continue. - # ``` - task :lock => gemfiles end - # rubocop:enable Style/MultilineBlockChain end