From 9e5951f211f27840c8552245d57fb6b102b28f0d Mon Sep 17 00:00:00 2001 From: jorg-vr Date: Tue, 3 Oct 2023 14:15:48 +0200 Subject: [PATCH] Avoid simultaneous reprocessing of a repository --- app/models/activity.rb | 1 + app/models/content_page.rb | 1 + app/models/exercise.rb | 1 + app/models/repository.rb | 55 +++++++++++++------ ...20_add_reprocess_status_to_repositories.rb | 6 ++ db/schema.rb | 5 +- test/factories/content_pages.rb | 1 + test/factories/exercises.rb | 1 + test/factories/repositories.rb | 20 ++++--- test/fixtures/exercises.yml | 1 + test/models/activity_test.rb | 1 + test/models/exercise_test.rb | 1 + test/models/repository_test.rb | 20 ++++--- 13 files changed, 77 insertions(+), 37 deletions(-) create mode 100644 db/migrate/20231003115520_add_reprocess_status_to_repositories.rb diff --git a/app/models/activity.rb b/app/models/activity.rb index 9409641565..011b2930c7 100644 --- a/app/models/activity.rb +++ b/app/models/activity.rb @@ -22,6 +22,7 @@ # description_nl_present :boolean default(FALSE) # description_en_present :boolean default(FALSE) # series_count :integer default(0), not null +# draft :boolean default(FALSE) # require 'pathname' diff --git a/app/models/content_page.rb b/app/models/content_page.rb index 5b40275d30..6136a20dea 100644 --- a/app/models/content_page.rb +++ b/app/models/content_page.rb @@ -22,6 +22,7 @@ # description_nl_present :boolean default(FALSE) # description_en_present :boolean default(FALSE) # series_count :integer default(0), not null +# draft :boolean default(FALSE) # class ContentPage < Activity diff --git a/app/models/exercise.rb b/app/models/exercise.rb index 23027bb738..2be76bfd3f 100644 --- a/app/models/exercise.rb +++ b/app/models/exercise.rb @@ -22,6 +22,7 @@ # description_nl_present :boolean default(FALSE) # description_en_present :boolean default(FALSE) # series_count :integer default(0), not null +# draft :boolean default(FALSE) # require 'pathname' diff --git a/app/models/repository.rb b/app/models/repository.rb index a9d0922ba0..5dbd6f1e28 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -2,15 +2,17 @@ # # Table name: repositories # -# id :integer not null, primary key -# name :string(255) -# remote :string(255) -# path :string(255) -# judge_id :integer -# created_at :datetime not null -# updated_at :datetime not null -# clone_status :integer default("queued"), not null -# featured :boolean default(FALSE) +# id :integer not null, primary key +# name :string(255) +# remote :string(255) +# path :string(255) +# judge_id :integer +# created_at :datetime not null +# updated_at :datetime not null +# clone_status :integer default("queued"), not null +# featured :boolean default(FALSE) +# reprocess_queued :boolean default(FALSE) +# reprocess_running :boolean default(FALSE) # require 'open3' require 'pathname' @@ -113,18 +115,35 @@ def clone_repo end def process_activities_email_errors_delayed(kwargs = {}) - delay(queue: 'git').process_activities_email_errors(kwargs) + return if reprocess_queued? + + update(reprocess_queued: true) + if reprocess_running? + delay(queue: 'git', run_at: 1.minute.from_now).process_activities_email_errors(kwargs) + else + delay(queue: 'git').process_activities_email_errors(kwargs) + end end def process_activities_email_errors(kwargs = {}) - kwargs[:user] = admins.first if kwargs.empty? && admins.any? - kwargs[:email] = Rails.application.config.dodona_email if kwargs.empty? - - process_activities - rescue AggregatedConfigErrors => e - ErrorMailer.json_error(e, **kwargs).deliver - rescue DodonaGitError => e - ErrorMailer.git_error(e, **kwargs).deliver + if reprocess_running? + delay(queue: 'git', run_at: 1.minute.from_now).process_activities_email_errors(kwargs) + return + end + + update(reprocess_queued: false, reprocess_running: true) + begin + kwargs[:user] = admins.first if kwargs.empty? && admins.any? + kwargs[:email] = Rails.application.config.dodona_email if kwargs.empty? + + process_activities + rescue AggregatedConfigErrors => e + ErrorMailer.json_error(e, **kwargs).deliver + rescue DodonaGitError => e + ErrorMailer.git_error(e, **kwargs).deliver + ensure + update(reprocess_running: false) + end end def process_activities diff --git a/db/migrate/20231003115520_add_reprocess_status_to_repositories.rb b/db/migrate/20231003115520_add_reprocess_status_to_repositories.rb new file mode 100644 index 0000000000..7c90ee141d --- /dev/null +++ b/db/migrate/20231003115520_add_reprocess_status_to_repositories.rb @@ -0,0 +1,6 @@ +class AddReprocessStatusToRepositories < ActiveRecord::Migration[7.0] + def change + add_column :repositories, :reprocess_queued, :boolean, default: false + add_column :repositories, :reprocess_running, :boolean, default: false + end +end diff --git a/db/schema.rb b/db/schema.rb index a501f8e4a3..763f29ed64 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2023_08_10_105908) do +ActiveRecord::Schema[7.0].define(version: 2023_10_03_115520) do create_table "active_storage_attachments", charset: "utf8mb4", collation: "utf8mb4_unicode_ci", force: :cascade do |t| t.string "name", null: false t.string "record_type", null: false @@ -59,6 +59,7 @@ t.boolean "description_nl_present", default: false t.boolean "description_en_present", default: false t.integer "series_count", default: 0, null: false + t.boolean "draft", default: false t.index ["judge_id"], name: "index_activities_on_judge_id" t.index ["name_nl"], name: "index_activities_on_name_nl" t.index ["path", "repository_id"], name: "index_activities_on_path_and_repository_id", unique: true @@ -392,6 +393,8 @@ t.datetime "updated_at", precision: nil, null: false t.integer "clone_status", default: 1, null: false t.boolean "featured", default: false + t.boolean "reprocess_queued", default: false + t.boolean "reprocess_running", default: false t.index ["judge_id"], name: "index_repositories_on_judge_id" t.index ["name"], name: "index_repositories_on_name", unique: true t.index ["path"], name: "index_repositories_on_path", unique: true diff --git a/test/factories/content_pages.rb b/test/factories/content_pages.rb index 64df42ed1f..aa87872f6e 100644 --- a/test/factories/content_pages.rb +++ b/test/factories/content_pages.rb @@ -22,6 +22,7 @@ # description_nl_present :boolean default(FALSE) # description_en_present :boolean default(FALSE) # series_count :integer default(0), not null +# draft :boolean default(FALSE) # require "#{File.dirname(__FILE__)}/../testhelpers/stub_helper.rb" diff --git a/test/factories/exercises.rb b/test/factories/exercises.rb index 1399693a9c..835cd4866f 100644 --- a/test/factories/exercises.rb +++ b/test/factories/exercises.rb @@ -22,6 +22,7 @@ # description_nl_present :boolean default(FALSE) # description_en_present :boolean default(FALSE) # series_count :integer default(0), not null +# draft :boolean default(FALSE) # require "#{File.dirname(__FILE__)}/../testhelpers/stub_helper.rb" diff --git a/test/factories/repositories.rb b/test/factories/repositories.rb index 7f45ca9bfa..f58c95831e 100644 --- a/test/factories/repositories.rb +++ b/test/factories/repositories.rb @@ -2,15 +2,17 @@ # # Table name: repositories # -# id :integer not null, primary key -# name :string(255) -# remote :string(255) -# path :string(255) -# judge_id :integer -# created_at :datetime not null -# updated_at :datetime not null -# clone_status :integer default("queued"), not null -# featured :boolean default(FALSE) +# id :integer not null, primary key +# name :string(255) +# remote :string(255) +# path :string(255) +# judge_id :integer +# created_at :datetime not null +# updated_at :datetime not null +# clone_status :integer default("queued"), not null +# featured :boolean default(FALSE) +# reprocess_queued :boolean default(FALSE) +# reprocess_running :boolean default(FALSE) # require "#{File.dirname(__FILE__)}/../testhelpers/stub_helper.rb" diff --git a/test/fixtures/exercises.yml b/test/fixtures/exercises.yml index 51040f7026..435b234b15 100644 --- a/test/fixtures/exercises.yml +++ b/test/fixtures/exercises.yml @@ -22,6 +22,7 @@ # description_nl_present :boolean default(FALSE) # description_en_present :boolean default(FALSE) # series_count :integer default(0), not null +# draft :boolean default(FALSE) # python_exercise: diff --git a/test/models/activity_test.rb b/test/models/activity_test.rb index 6e3d02ff59..1a8103bdad 100644 --- a/test/models/activity_test.rb +++ b/test/models/activity_test.rb @@ -22,6 +22,7 @@ # description_nl_present :boolean default(FALSE) # description_en_present :boolean default(FALSE) # series_count :integer default(0), not null +# draft :boolean default(FALSE) # require 'test_helper' diff --git a/test/models/exercise_test.rb b/test/models/exercise_test.rb index 541f8a7571..fc94dfec23 100644 --- a/test/models/exercise_test.rb +++ b/test/models/exercise_test.rb @@ -22,6 +22,7 @@ # description_nl_present :boolean default(FALSE) # description_en_present :boolean default(FALSE) # series_count :integer default(0), not null +# draft :boolean default(FALSE) # require 'test_helper' diff --git a/test/models/repository_test.rb b/test/models/repository_test.rb index 66c5d068d6..d7adf8cd03 100644 --- a/test/models/repository_test.rb +++ b/test/models/repository_test.rb @@ -2,15 +2,17 @@ # # Table name: repositories # -# id :integer not null, primary key -# name :string(255) -# remote :string(255) -# path :string(255) -# judge_id :integer -# created_at :datetime not null -# updated_at :datetime not null -# clone_status :integer default("queued"), not null -# featured :boolean default(FALSE) +# id :integer not null, primary key +# name :string(255) +# remote :string(255) +# path :string(255) +# judge_id :integer +# created_at :datetime not null +# updated_at :datetime not null +# clone_status :integer default("queued"), not null +# featured :boolean default(FALSE) +# reprocess_queued :boolean default(FALSE) +# reprocess_running :boolean default(FALSE) # require 'test_helper'