From 7dbda40695969eb6c4a84d885cede98bc0599a9c Mon Sep 17 00:00:00 2001
From: Brandon Dunne <brandondunne@hotmail.com>
Date: Thu, 24 Jan 2019 16:32:25 -0500
Subject: [PATCH] Check the remote schema_migrations table if the migration is
 not listed

If a column is added to MiqRegion, that causes replication to stop.  With
replication stopped we sit in the loop forever waiting for the MiqRegion
record for the remote region to be updated with the migrations_ran.  In
this case, we need to check the schema_migrations table on the remote
region directly.

Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1668800
---
 lib/extensions/ar_migration.rb           | 19 ++++++++++++++++++-
 spec/lib/extensions/ar_migration_spec.rb | 23 ++++++++++++++++++++++-
 2 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/lib/extensions/ar_migration.rb b/lib/extensions/ar_migration.rb
index e952bad150b4..4dbc77ae397d 100644
--- a/lib/extensions/ar_migration.rb
+++ b/lib/extensions/ar_migration.rb
@@ -64,7 +64,24 @@ def wait_for_remote_region_migration(wait_time = 1)
     private
 
     def wait_for_migration?
-      migrations_column_present? ? !region.migrations_ran&.include?(version) : false
+      migrations_column_present? ? !remote_region_migrated? : false
+    end
+
+    # NOTE: Check the local table first, but a migration can cause MiqRegion replication to stop (i.e. adding a new column to miq_regions).
+    # In that case, the global MiqRegion record for the remote region will not have the migration timestamp yet, so we need to check the
+    # schema_migrations table in the remote region directly.
+    def remote_region_migrated?
+      version_exists_in_replicated_region? || version_exists_on_remote_region?
+    end
+
+    def version_exists_in_replicated_region?
+      region.migrations_ran&.include?(version)
+    end
+
+    def version_exists_on_remote_region?
+      subscription.send(:with_remote_connection) do |connection|
+        connection.select_values("SELECT 1 FROM schema_migrations WHERE version = '#{version}' LIMIT 1")
+      end.any?
     end
 
     def migrations_column_present?
diff --git a/spec/lib/extensions/ar_migration_spec.rb b/spec/lib/extensions/ar_migration_spec.rb
index fc823f5af3fe..045ae4468415 100644
--- a/spec/lib/extensions/ar_migration_spec.rb
+++ b/spec/lib/extensions/ar_migration_spec.rb
@@ -80,9 +80,10 @@
         end
       end
 
-      it "sleeps until the migration is added" do
+      it "sleeps until the migration is replicated up" do
         allow(subject).to receive(:restart_subscription)
         allow(subject.region).to receive(:reload)
+        allow(subscription).to receive(:with_remote_connection).and_return([])
 
         subject.region.update_attributes!(:migrations_ran => nil)
 
@@ -105,6 +106,26 @@
         t = t.join(5)
         expect(t.status).to be false
       end
+
+      it "when migrations_ran is nil, it checks the schema_migrations table on the remote region" do
+        expect(subject).not_to receive(:restart_subscription)
+        expect(subject.region).not_to receive(:reload)
+        expect(subscription).to receive(:with_remote_connection).and_return([version])
+
+        subject.region.update_attributes!(:migrations_ran => nil)
+
+        subject.wait_for_remote_region_migration(0)
+      end
+
+      it "when migrations_ran does not include that expected migration, it checks the schema_migrations on the remote region" do
+        expect(subject).not_to receive(:restart_subscription)
+        expect(subject.region).not_to receive(:reload)
+        expect(subscription).to receive(:with_remote_connection).and_return([version])
+
+        subject.region.update_attributes!(:migrations_ran => ["1234", "5678"])
+
+        subject.wait_for_remote_region_migration(0)
+      end
     end
   end
 end