From 549336b0a215e02bc9aa9a8477ba4271587304e7 Mon Sep 17 00:00:00 2001
From: Ben Halpern <bendhalpern@gmail.com>
Date: Tue, 12 Oct 2021 16:27:13 -0400
Subject: [PATCH] Delete old retained emails after certain period (#14949)

* Delete old retained emails after certain period

* Update app/models/email_message.rb

Co-authored-by: Jamie Gaskins <jamie@forem.com>

* Update spec/workers/emails/remove_old_emails_worker_spec.rb

Co-authored-by: Jamie Gaskins <jamie@forem.com>

* Update app/models/email_message.rb

Co-authored-by: Jamie Gaskins <jamie@forem.com>

Co-authored-by: Jamie Gaskins <jamie@forem.com>
---
 app/models/email_message.rb                   | 19 +++++++++++++++++++
 .../emails/remove_old_emails_worker.rb        | 11 +++++++++++
 config/schedule.yml                           |  4 ++++
 spec/models/email_message_spec.rb             |  8 ++++++++
 .../emails/remove_old_emails_worker_spec.rb   | 15 +++++++++++++++
 5 files changed, 57 insertions(+)
 create mode 100644 app/workers/emails/remove_old_emails_worker.rb
 create mode 100644 spec/workers/emails/remove_old_emails_worker_spec.rb

diff --git a/app/models/email_message.rb b/app/models/email_message.rb
index eadd8b815a516..4f7ac9e08535d 100644
--- a/app/models/email_message.rb
+++ b/app/models/email_message.rb
@@ -11,4 +11,23 @@ def self.find_for_reports(feedback_message_ids)
     select(:to, :subject, :content, :utm_campaign, :feedback_message_id)
       .where(feedback_message_id: feedback_message_ids)
   end
+
+  def self.fast_destroy_old_retained_email_deliveries(destroy_before_timestamp = 3.months.ago)
+    # We remove email delivery records periodically, except some we retain long term.
+    # We generally want to retain emails directly sent by human admins.
+    # The only email currently sent manually are those that are tied directly to a feedback message.
+    sql = <<~SQL
+      DELETE FROM ahoy_messages
+      WHERE ahoy_messages.id IN (
+        SELECT ahoy_messages.id
+        FROM ahoy_messages
+        WHERE sent_at < ? AND feedback_message_id IS NULL
+        LIMIT 50000
+      )
+    SQL
+
+    email_sql = EmailMessage.sanitize_sql([sql, destroy_before_timestamp])
+
+    BulkSqlDelete.delete_in_batches(email_sql)
+  end
 end
diff --git a/app/workers/emails/remove_old_emails_worker.rb b/app/workers/emails/remove_old_emails_worker.rb
new file mode 100644
index 0000000000000..04b8e8b36a308
--- /dev/null
+++ b/app/workers/emails/remove_old_emails_worker.rb
@@ -0,0 +1,11 @@
+module Emails
+  class RemoveOldEmailsWorker
+    include Sidekiq::Worker
+
+    sidekiq_options queue: :low_priority, retry: 10
+
+    def perform
+      EmailMessage.fast_destroy_old_retained_email_deliveries
+    end
+  end
+end
diff --git a/config/schedule.yml b/config/schedule.yml
index e4c1fbc996b58..120de61dff593 100644
--- a/config/schedule.yml
+++ b/config/schedule.yml
@@ -120,6 +120,10 @@ remove_old_notifications:
   description: "Deletes old notifications from the database (runs daily at 05:00 UTC)"
   cron: "0 5 * * *"
   class: "Notifications::RemoveOldNotificationsWorker"
+remove_old_emails:
+  description: "Deletes old emails we don't need to retain from the database (runs daily at 06:00 UTC)"
+  cron: "0 6 * * *"
+  class: "Emails::RemoveOldEmailsWorker"
 sync_credits_counter_cache:
   description: "Sychronizes counter caches for credits (runs daily at 16:00 UTC)"
   cron: "0 16 * * *"
diff --git a/spec/models/email_message_spec.rb b/spec/models/email_message_spec.rb
index eeda199499c2e..ee7b12713684b 100644
--- a/spec/models/email_message_spec.rb
+++ b/spec/models/email_message_spec.rb
@@ -6,4 +6,12 @@
 
     it { is_expected.to belong_to(:feedback_message).optional }
   end
+
+  describe "#fast_destroy_old_notifications" do
+    it "bulk deletes emails older than given timestamp" do
+      allow(BulkSqlDelete).to receive(:delete_in_batches)
+      described_class.fast_destroy_old_retained_email_deliveries("a_time")
+      expect(BulkSqlDelete).to have_received(:delete_in_batches).with(a_string_including("< 'a_time'"))
+    end
+  end
 end
diff --git a/spec/workers/emails/remove_old_emails_worker_spec.rb b/spec/workers/emails/remove_old_emails_worker_spec.rb
new file mode 100644
index 0000000000000..0a1bdb464544f
--- /dev/null
+++ b/spec/workers/emails/remove_old_emails_worker_spec.rb
@@ -0,0 +1,15 @@
+require "rails_helper"
+
+RSpec.describe Emails::RemoveOldEmailsWorker, type: :worker do
+  include_examples "#enqueues_on_correct_queue", "low_priority"
+
+  describe "#perform" do
+    let(:worker) { subject }
+
+    it "fast destroys notifications" do
+      allow(EmailMessage).to receive(:fast_destroy_old_retained_email_deliveries)
+      worker.perform
+      expect(EmailMessage).to have_received(:fast_destroy_old_retained_email_deliveries)
+    end
+  end
+end