Avoid simultaneous reprocessing of a repository

dodona-edu · Oct 3, 2023 · 9e5951f · 9e5951f
1 parent 12f2329
commit 9e5951f
Show file tree

Hide file tree

Showing 13 changed files with 77 additions and 37 deletions.
diff --git a/app/models/activity.rb b/app/models/activity.rb
@@ -22,6 +22,7 @@
 #  description_nl_present  :boolean          default(FALSE)
 #  description_en_present  :boolean          default(FALSE)
 #  series_count            :integer          default(0), not null
+#  draft                   :boolean          default(FALSE)
 #
 
 require 'pathname'

diff --git a/app/models/content_page.rb b/app/models/content_page.rb
@@ -22,6 +22,7 @@
 #  description_nl_present  :boolean          default(FALSE)
 #  description_en_present  :boolean          default(FALSE)
 #  series_count            :integer          default(0), not null
+#  draft                   :boolean          default(FALSE)
 #
 
 class ContentPage < Activity

diff --git a/app/models/exercise.rb b/app/models/exercise.rb
@@ -22,6 +22,7 @@
 #  description_nl_present  :boolean          default(FALSE)
 #  description_en_present  :boolean          default(FALSE)
 #  series_count            :integer          default(0), not null
+#  draft                   :boolean          default(FALSE)
 #
 
 require 'pathname'

diff --git a/app/models/repository.rb b/app/models/repository.rb
@@ -2,15 +2,17 @@
 #
 # Table name: repositories
 #
-#  id           :integer          not null, primary key
-#  name         :string(255)
-#  remote       :string(255)
-#  path         :string(255)
-#  judge_id     :integer
-#  created_at   :datetime         not null
-#  updated_at   :datetime         not null
-#  clone_status :integer          default("queued"), not null
-#  featured     :boolean          default(FALSE)
+#  id                :integer          not null, primary key
+#  name              :string(255)
+#  remote            :string(255)
+#  path              :string(255)
+#  judge_id          :integer
+#  created_at        :datetime         not null
+#  updated_at        :datetime         not null
+#  clone_status      :integer          default("queued"), not null
+#  featured          :boolean          default(FALSE)
+#  reprocess_queued  :boolean          default(FALSE)
+#  reprocess_running :boolean          default(FALSE)
 #
 require 'open3'
 require 'pathname'
@@ -113,18 +115,35 @@ def clone_repo
   end
 
   def process_activities_email_errors_delayed(kwargs = {})
-    delay(queue: 'git').process_activities_email_errors(kwargs)
+    return if reprocess_queued?
+
+    update(reprocess_queued: true)
+    if reprocess_running?
+      delay(queue: 'git', run_at: 1.minute.from_now).process_activities_email_errors(kwargs)
+    else
+      delay(queue: 'git').process_activities_email_errors(kwargs)
+    end
   end
 
   def process_activities_email_errors(kwargs = {})
-    kwargs[:user] = admins.first if kwargs.empty? && admins.any?
-    kwargs[:email] = Rails.application.config.dodona_email if kwargs.empty?
-
-    process_activities
-  rescue AggregatedConfigErrors => e
-    ErrorMailer.json_error(e, **kwargs).deliver
-  rescue DodonaGitError => e
-    ErrorMailer.git_error(e, **kwargs).deliver
+    if reprocess_running?
+      delay(queue: 'git', run_at: 1.minute.from_now).process_activities_email_errors(kwargs)
+      return
+    end
+
+    update(reprocess_queued: false, reprocess_running: true)
+    begin
+      kwargs[:user] = admins.first if kwargs.empty? && admins.any?
+      kwargs[:email] = Rails.application.config.dodona_email if kwargs.empty?
+
+      process_activities
+    rescue AggregatedConfigErrors => e
+      ErrorMailer.json_error(e, **kwargs).deliver
+    rescue DodonaGitError => e
+      ErrorMailer.git_error(e, **kwargs).deliver
+    ensure
+      update(reprocess_running: false)
+    end
   end
 
   def process_activities

diff --git a/db/migrate/20231003115520_add_reprocess_status_to_repositories.rb b/db/migrate/20231003115520_add_reprocess_status_to_repositories.rb
@@ -0,0 +1,6 @@
+class AddReprocessStatusToRepositories < ActiveRecord::Migration[7.0]
+  def change
+    add_column :repositories, :reprocess_queued, :boolean, default: false
+    add_column :repositories, :reprocess_running, :boolean, default: false
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2023_08_10_105908) do
+ActiveRecord::Schema[7.0].define(version: 2023_10_03_115520) do
   create_table "active_storage_attachments", charset: "utf8mb4", collation: "utf8mb4_unicode_ci", force: :cascade do |t|
     t.string "name", null: false
     t.string "record_type", null: false
@@ -59,6 +59,7 @@
     t.boolean "description_nl_present", default: false
     t.boolean "description_en_present", default: false
     t.integer "series_count", default: 0, null: false
+    t.boolean "draft", default: false
     t.index ["judge_id"], name: "index_activities_on_judge_id"
     t.index ["name_nl"], name: "index_activities_on_name_nl"
     t.index ["path", "repository_id"], name: "index_activities_on_path_and_repository_id", unique: true
@@ -392,6 +393,8 @@
     t.datetime "updated_at", precision: nil, null: false
     t.integer "clone_status", default: 1, null: false
     t.boolean "featured", default: false
+    t.boolean "reprocess_queued", default: false
+    t.boolean "reprocess_running", default: false
     t.index ["judge_id"], name: "index_repositories_on_judge_id"
     t.index ["name"], name: "index_repositories_on_name", unique: true
     t.index ["path"], name: "index_repositories_on_path", unique: true

diff --git a/test/factories/content_pages.rb b/test/factories/content_pages.rb
@@ -22,6 +22,7 @@
 #  description_nl_present  :boolean          default(FALSE)
 #  description_en_present  :boolean          default(FALSE)
 #  series_count            :integer          default(0), not null
+#  draft                   :boolean          default(FALSE)
 #
 
 require "#{File.dirname(__FILE__)}/../testhelpers/stub_helper.rb"

diff --git a/test/factories/exercises.rb b/test/factories/exercises.rb
@@ -22,6 +22,7 @@
 #  description_nl_present  :boolean          default(FALSE)
 #  description_en_present  :boolean          default(FALSE)
 #  series_count            :integer          default(0), not null
+#  draft                   :boolean          default(FALSE)
 #
 
 require "#{File.dirname(__FILE__)}/../testhelpers/stub_helper.rb"

diff --git a/test/factories/repositories.rb b/test/factories/repositories.rb
@@ -2,15 +2,17 @@
 #
 # Table name: repositories
 #
-#  id           :integer          not null, primary key
-#  name         :string(255)
-#  remote       :string(255)
-#  path         :string(255)
-#  judge_id     :integer
-#  created_at   :datetime         not null
-#  updated_at   :datetime         not null
-#  clone_status :integer          default("queued"), not null
-#  featured     :boolean          default(FALSE)
+#  id                :integer          not null, primary key
+#  name              :string(255)
+#  remote            :string(255)
+#  path              :string(255)
+#  judge_id          :integer
+#  created_at        :datetime         not null
+#  updated_at        :datetime         not null
+#  clone_status      :integer          default("queued"), not null
+#  featured          :boolean          default(FALSE)
+#  reprocess_queued  :boolean          default(FALSE)
+#  reprocess_running :boolean          default(FALSE)
 #
 
 require "#{File.dirname(__FILE__)}/../testhelpers/stub_helper.rb"

diff --git a/test/fixtures/exercises.yml b/test/fixtures/exercises.yml
@@ -22,6 +22,7 @@
 #  description_nl_present  :boolean          default(FALSE)
 #  description_en_present  :boolean          default(FALSE)
 #  series_count            :integer          default(0), not null
+#  draft                   :boolean          default(FALSE)
 #
 
 python_exercise:

diff --git a/test/models/activity_test.rb b/test/models/activity_test.rb
@@ -22,6 +22,7 @@
 #  description_nl_present  :boolean          default(FALSE)
 #  description_en_present  :boolean          default(FALSE)
 #  series_count            :integer          default(0), not null
+#  draft                   :boolean          default(FALSE)
 #
 
 require 'test_helper'

diff --git a/test/models/exercise_test.rb b/test/models/exercise_test.rb
@@ -22,6 +22,7 @@
 #  description_nl_present  :boolean          default(FALSE)
 #  description_en_present  :boolean          default(FALSE)
 #  series_count            :integer          default(0), not null
+#  draft                   :boolean          default(FALSE)
 #
 
 require 'test_helper'

diff --git a/test/models/repository_test.rb b/test/models/repository_test.rb
@@ -2,15 +2,17 @@
 #
 # Table name: repositories
 #
-#  id           :integer          not null, primary key
-#  name         :string(255)
-#  remote       :string(255)
-#  path         :string(255)
-#  judge_id     :integer
-#  created_at   :datetime         not null
-#  updated_at   :datetime         not null
-#  clone_status :integer          default("queued"), not null
-#  featured     :boolean          default(FALSE)
+#  id                :integer          not null, primary key
+#  name              :string(255)
+#  remote            :string(255)
+#  path              :string(255)
+#  judge_id          :integer
+#  created_at        :datetime         not null
+#  updated_at        :datetime         not null
+#  clone_status      :integer          default("queued"), not null
+#  featured          :boolean          default(FALSE)
+#  reprocess_queued  :boolean          default(FALSE)
+#  reprocess_running :boolean          default(FALSE)
 #
 
 require 'test_helper'