Delete old retained emails after certain period (forem#14949)

* Delete old retained emails after certain period

* Update app/models/email_message.rb

Co-authored-by: Jamie Gaskins <[email protected]>

* Update spec/workers/emails/remove_old_emails_worker_spec.rb

Co-authored-by: Jamie Gaskins <[email protected]>

* Update app/models/email_message.rb

Co-authored-by: Jamie Gaskins <[email protected]>

Co-authored-by: Jamie Gaskins <[email protected]>

app/models/email_message.rb

@@ -11,4 +11,23 @@ def self.find_for_reports(feedback_message_ids)
     select(:to, :subject, :content, :utm_campaign, :feedback_message_id)
       .where(feedback_message_id: feedback_message_ids)
   end
+
+  def self.fast_destroy_old_retained_email_deliveries(destroy_before_timestamp = 3.months.ago)
+    # We remove email delivery records periodically, except some we retain long term.
+    # We generally want to retain emails directly sent by human admins.
+    # The only email currently sent manually are those that are tied directly to a feedback message.
+    sql = <<~SQL
+      DELETE FROM ahoy_messages
+      WHERE ahoy_messages.id IN (
+        SELECT ahoy_messages.id
+        FROM ahoy_messages
+        WHERE sent_at < ? AND feedback_message_id IS NULL
+        LIMIT 50000
+      )
+    SQL
+
+    email_sql = EmailMessage.sanitize_sql([sql, destroy_before_timestamp])
+
+    BulkSqlDelete.delete_in_batches(email_sql)
+  end
 end

app/workers/emails/remove_old_emails_worker.rb

@@ -0,0 +1,11 @@
+module Emails
+  class RemoveOldEmailsWorker
+    include Sidekiq::Worker
+
+    sidekiq_options queue: :low_priority, retry: 10
+
+    def perform
+      EmailMessage.fast_destroy_old_retained_email_deliveries
+    end
+  end
+end

config/schedule.yml

@@ -120,6 +120,10 @@ remove_old_notifications:
   description: "Deletes old notifications from the database (runs daily at 05:00 UTC)"
   cron: "0 5 * * *"
   class: "Notifications::RemoveOldNotificationsWorker"
+remove_old_emails:
+  description: "Deletes old emails we don't need to retain from the database (runs daily at 06:00 UTC)"
+  cron: "0 6 * * *"
+  class: "Emails::RemoveOldEmailsWorker"
 sync_credits_counter_cache:
   description: "Sychronizes counter caches for credits (runs daily at 16:00 UTC)"
   cron: "0 16 * * *"

spec/models/email_message_spec.rb

@@ -6,4 +6,12 @@
 
     it { is_expected.to belong_to(:feedback_message).optional }
   end
+
+  describe "#fast_destroy_old_notifications" do
+    it "bulk deletes emails older than given timestamp" do
+      allow(BulkSqlDelete).to receive(:delete_in_batches)
+      described_class.fast_destroy_old_retained_email_deliveries("a_time")
+      expect(BulkSqlDelete).to have_received(:delete_in_batches).with(a_string_including("< 'a_time'"))
+    end
+  end
 end

spec/workers/emails/remove_old_emails_worker_spec.rb

@@ -0,0 +1,15 @@
+require "rails_helper"
+
+RSpec.describe Emails::RemoveOldEmailsWorker, type: :worker do
+  include_examples "#enqueues_on_correct_queue", "low_priority"
+
+  describe "#perform" do
+    let(:worker) { subject }
+
+    it "fast destroys notifications" do
+      allow(EmailMessage).to receive(:fast_destroy_old_retained_email_deliveries)
+      worker.perform
+      expect(EmailMessage).to have_received(:fast_destroy_old_retained_email_deliveries)
+    end
+  end
+end