diff --git a/lib/tasks/update_public_index.rake b/lib/tasks/update_public_index.rake index c24226f6..60056c3d 100644 --- a/lib/tasks/update_public_index.rake +++ b/lib/tasks/update_public_index.rake @@ -1,3 +1,7 @@ +# This can be quite difficult to debug so in the tests +# there is a commented out test that will help you +# generate all sorts of scenarios on top of the tests themselves + desc 'Update Outpost public index copies in the Mongo API' task :update_public_index => :environment do # Turn off logging for this rake task, otherwise it just fills up our logs @@ -7,14 +11,14 @@ task :update_public_index => :environment do Mongo::Logger.logger.level = Logger::FATAL - puts "⏰ Connecting to mongo database..." + puts "⏰ Connecting to mongo database...\n\n" client = Mongo::Client.new(ENV["DB_URI"] || 'mongodb://root:password@localhost:27017/outpost_development?authSource=admin', { retry_writes: false }) collection = client.database[:indexed_services] approved_count, unapproved_count, deleted_count, deleted_skipped_count = 0, 0, 0, 0 active_count, temporarily_closed_count, scheduled_count, archived_count, expired_count, invisible_count, marked_for_deletion_count, pending_count = 0, 0, 0, 0, 0, 0, 0, 0 - active_ids, temporarily_closed_ids, scheduled_ids, archived_ids, expired_ids, invisible_ids, marked_for_deletion_ids, pending_ids = [], [], [], [], [], [], [], [] + active_ids, temporarily_closed_ids, scheduled_ids, archived_ids, expired_ids, invisible_ids, marked_for_deletion_ids, pending_ids, archived_skipped_ids, expired_skipped_ids, invisible_skipped_ids, marked_for_deletion_skipped_ids, pending_skipped_ids = [], [], [], [], [], [], [], [], [], [], [], [], [] Service.find_each do |service| case service.status @@ -25,7 +29,7 @@ task :update_public_index => :environment do IndexedServicesSerializer.new(service).as_json, { upsert: true }) active_count += 1 - puts "ACTIVE: #{service.name} indexed" + puts "🟢 ACTV: #{service.id} #{service.name} indexed" active_ids << service.id # temporarily closed services are all indexed @@ -34,7 +38,7 @@ task :update_public_index => :environment do IndexedServicesSerializer.new(service).as_json, { upsert: true }) temporarily_closed_count += 1 - puts "TEMPORARILY CLOSED: #{service.name} indexed" + puts "🟢 TEMP: #{service.id} #{service.name} indexed" temporarily_closed_ids << service.id # scheduled services are all indexed - the API determines if they're returned or not @@ -43,7 +47,7 @@ task :update_public_index => :environment do IndexedServicesSerializer.new(service).as_json, { upsert: true }) scheduled_count += 1 - puts "SCHEDULED: #{service.name} indexed" + puts "🟢 SCHD: #{service.id} #{service.name} indexed" scheduled_ids << service.id @@ -53,9 +57,10 @@ task :update_public_index => :environment do archived_count += 1 if deleted_archived archived_ids << service.id - puts "🗑 ARCHIVED: #{service.name} deleted" + puts "🔴 ARCH: #{service.id} #{service.name} DELETED" else - puts "⚠️ ARCHIVED: #{service.name} not found in index, skipping" + archived_skipped_ids << service.id + puts "🟡 ARCH: #{service.id} #{service.name} not found in index, skipping" end # expired services are removed from the index @@ -64,20 +69,22 @@ task :update_public_index => :environment do expired_count += 1 if deleted_expired expired_ids << service.id - puts "🗑 EXPIRED: #{service.name} deleted" + puts "🔴 EXPD: #{service.id} #{service.name} DELETED" else - puts "⚠️ EXPIRED: #{service.name} not found in index, skipping" + expired_skipped_ids << service.id + puts "🟡 EXPD: #{service.id} #{service.name} not found in index, skipping" end # invisible services are removed from the index when 'invisible' deleted_invisible = collection.find_one_and_delete({ id: service.id }) - deleted_count += 1 + invisible_count += 1 if deleted_invisible invisible_ids << service.id - puts "🗑 INVISIBLE: #{service.name} deleted" + puts "🔴 INVS: #{service.id} #{service.name} DELETED" else - puts "⚠️ INVISIBLE: #{service.name} not found in index, skipping" + invisible_skipped_ids << service.id + puts "🟡 INVS: #{service.id} #{service.name} not found in index, skipping" end # marked for deletion definitely get removed from the index @@ -86,21 +93,34 @@ task :update_public_index => :environment do marked_for_deletion_count += 1 if deleted_marked_for_deletion marked_for_deletion_ids << service.id - puts "🗑 MARKED FOR DELETION: #{service.name} deleted" + puts "🔴 MKDL: #{service.id} #{service.name} DELETED" else - puts "⚠️ MARKED FOR DELETION: #{service.name} not found in index, skipping" + marked_for_deletion_skipped_ids << service.id + puts "🟡 MKDL: #{service.id} #{service.name} not found in index, skipping" end - # if status is pending we work off last approved snapshot, if it exists and is visible or we make a snapshot to make it visible + # if status is pending we work off last version where approved === true, if it exists and is visible or we make a snapshot to make it visible when 'pending' + pending_count += 1 + + puts "⚪️ PEND: #{service.id} #{service.name}" + puts "\s\s\s\s\s- Service has #{service.versions.count} versions" + puts "\s\s\s\s\s- Service has #{service.last_approved_snapshot ? 'a' : 'no'} previously approved snapshot" + if service.last_approved_snapshot + puts "\s\s\s\s\s\s\s- approved snapshot is #{service.last_approved_snapshot.object['visible'] == true ? 'visible' : 'invisible'}" + puts "\s\s\s\s\s\s\s- approved snapshot is #{service.last_approved_snapshot.object['discarded_at'].blank? ? 'not discarded' : 'discarded'}" + end + approved_alternative = service.last_approved_snapshot unless approved_alternative - puts "🚨 No alternative approved snapshot of #{service.name} exists. Skipping." + pending_skipped_ids << service.id + puts "\s\s\s\s\s- 🟡 No approved version for #{service.id} #{service.name} exists. Skipping." next end unless approved_alternative.object['visible'] == true && approved_alternative.object['discarded_at'].blank? - puts "🚨 Approved snapshot of #{service.name} is not publicly visible. Skipping." + pending_skipped_ids << service.id + puts "\s\s\s\s\s- 🟠 Approved version of #{service.id} #{service.name} exists but it is not publicly visible. Skipping." next end @@ -108,35 +128,34 @@ task :update_public_index => :environment do collection.find_one_and_update({ id: service.id }, IndexedServicesSerializer.new(snapshot).as_json, { upsert: true }) - puts "🤔 Alternative approved snapshot of #{service.name} indexed" - pending_count += 1 + puts "\s\s\s\s\s- 🟢 Last approved version (id: #{approved_alternative.id}) of #{service.id} #{service.name} indexed" pending_ids << service.id end end - - # check if we missed any entries - indexed_ids = active_ids + temporarily_closed_ids + scheduled_ids - missed_services = collection.find({ id: { '$nin': indexed_ids } }) + # check if we missed any entries, best to be safe and remove them + indexed_ids = active_ids + temporarily_closed_ids + scheduled_ids + pending_ids + missed_services = collection.find({ id: { '$nin': indexed_ids.sort } }) - if missed_services.count > 0 + if missed_services and missed_services.count > 0 puts "\n\n" - puts "#{missed_services.count} missed services, deleting..." + puts "#{missed_services.count} services exist in mongo but not in outpost, deleting..." missed_service_ids = missed_services.map { |service| service['id'] } delete_missed_services = collection.delete_many({ id: { '$in': missed_service_ids } }) - delete_missed_services.each do |service| - puts "🗑 #{service.name} deleted" - end - puts "#{delete_missed_services.deleted_count} missed services deleted." + puts "#{delete_missed_services.deleted_count} unaccounted for services deleted." + else + puts "\n\n" + puts "No unaccounted for services left in the index." end + skipped_ids = archived_skipped_ids + expired_skipped_ids + invisible_skipped_ids + marked_for_deletion_skipped_ids + pending_skipped_ids deleted_ids = archived_ids + expired_ids + invisible_ids + marked_for_deletion_ids - puts "\n\n" puts "🏁🏁 SUMMARY 🏁🏁" - puts " 👉 #{indexed_ids.length + pending_ids.length} updated or created" + puts " 👉 #{indexed_ids.length} updated or created" + puts " 👉 #{skipped_ids.length} skipped" puts " 👉 #{deleted_ids.length} deleted" puts "\n\n\n" @@ -154,5 +173,5 @@ task :update_public_index => :environment do puts "\n\n\n" puts "Pending Services" - puts " 👉 #{pending_count} pending services, #{pending_ids.length} created." + puts " 👉 #{pending_count} pending services, #{pending_ids.length} created or updated." end diff --git a/spec/tasks/update_public_index_spec.rb b/spec/tasks/update_public_index_spec.rb new file mode 100644 index 00000000..88f1f3ca --- /dev/null +++ b/spec/tasks/update_public_index_spec.rb @@ -0,0 +1,269 @@ +# spec/tasks/update_public_index_spec.rb +require 'rails_helper' +require 'rake' + + +describe 'update_public_index task' do + + let(:task) { Rake::Task['update_public_index'] } + let(:collection) { double('indexed_services') } + + before do + Rake.application.rake_require('tasks/update_public_index') + Rake::Task.define_task(:environment) + + database = double('Database') + allow(database).to receive(:[]).with(:indexed_services).and_return(collection) + allow(Mongo::Client).to receive(:new).and_return(double('Mongo::Client', database: database)) + end + + after do + task.reenable # Re-enable the task after each test + end + + context 'when adding services to the Mongo database' do + let!(:service_active) { FactoryBot.create_list(:service, 3) } + let!(:service_temporarily_closed) { FactoryBot.create_list(:service, 3, temporarily_closed: true) } + let!(:service_scheduled) { FactoryBot.create_list(:service, 3, visible_from: Date.today + 2) } + + + + before do + service_active.each do |service| + serialized_service = IndexedServicesSerializer.new(service).as_json + allow(collection).to receive(:find_one_and_update).with( + { id: service.id }, + serialized_service, + { upsert: true } + ) + end + + service_temporarily_closed.each do |service| + serialized_service = IndexedServicesSerializer.new(service).as_json + allow(collection).to receive(:find_one_and_update).with( + { id: service.id }, + serialized_service, + { upsert: true } + ) + end + + service_scheduled.each do |service| + serialized_service = IndexedServicesSerializer.new(service).as_json + allow(collection).to receive(:find_one_and_update).with( + { id: service.id }, + serialized_service, + { upsert: true } + ) + end + + indexed_ids = service_active.map(&:id) + service_temporarily_closed.map(&:id) + service_scheduled.map(&:id) + missed_services = [ + { 'id' => 'missed_service_1' }, + { 'id' => 'missed_service_2' } + ] + allow(collection).to receive(:find).with({ id: { '$nin': indexed_ids.sort } }).and_return(missed_services) + allow(collection).to receive(:delete_many).with({ id: { '$in': ['missed_service_1', 'missed_service_2'] } }).and_return(double('result', deleted_count: 2)) + end + + it 'adds services to the Mongo database and removes any it has missed' do + expect { task.invoke }.to output( + match(/2 services exist in mongo but not in outpost, deleting.../) + .and match(/2 unaccounted for services deleted./) + .and match(/ 👉 9 updated or created/) + .and match(/ 👉 0 skipped/) + .and match(/ 👉 0 deleted/) + .and match(/ 👉 3 active services, 3 created or updated/) + .and match(/ 👉 3 temporarily_closed services, 3 created or updated/) + .and match(/ 👉 3 scheduled services, 3 created or updated/) + .and match(/ 👉 0 archived services, 0 deleted/) + .and match(/ 👉 0 expired services, 0 deleted/) + .and match(/ 👉 0 invisible services, 0 deleted/) + .and match(/ 👉 0 marked_for_deletion services, 0 deleted/) + .and match(/ 👉 0 pending services, 0 created or updated./) + ).to_stdout + end + + end + + context 'when removing services from the Mongo database' do + let!(:service_archived) { FactoryBot.create_list(:service, 3, discarded_at: 1.day.ago) } + let!(:service_expired) { FactoryBot.create_list(:service, 3, visible_to: Date.today - 2) } + let!(:service_invisible) { FactoryBot.create_list(:service, 3, visible: false) } + let!(:service_marked_for_deletion) { FactoryBot.create_list(:service, 3, marked_for_deletion: Time.now) } + + + before do + service_archived.each do |service| + allow(collection).to receive(:find_one_and_delete).with( + { id: service.id } + ).and_return(service.attributes) + end + + service_expired.each do |service| + allow(collection).to receive(:find_one_and_delete).with( + { id: service.id } + ).and_return(service.attributes) + end + + service_invisible.each do |service| + allow(collection).to receive(:find_one_and_delete).with( + { id: service.id } + ).and_return(service.attributes) + end + + service_marked_for_deletion.each do |service| + allow(collection).to receive(:find_one_and_delete).with( + { id: service.id } + ).and_return(service.attributes) + end + + allow(collection).to receive(:find).with({ id: { '$nin': [] } }) + end + + + it 'removes services from the Mongo database' do + expect { task.invoke }.to output( + match(/No unaccounted for services left in the index./) + .and match(/ 👉 0 updated or created/) + .and match(/ 👉 0 skipped/) + .and match(/ 👉 12 deleted/) + .and match(/ 👉 0 active services, 0 created or updated/) + .and match(/ 👉 0 temporarily_closed services, 0 created or updated/) + .and match(/ 👉 0 scheduled services, 0 created or updated/) + .and match(/ 👉 3 archived services, 3 deleted/) + .and match(/ 👉 3 expired services, 3 deleted/) + .and match(/ 👉 3 invisible services, 3 deleted/) + .and match(/ 👉 3 marked_for_deletion services, 3 deleted/) + .and match(/ 👉 0 pending services, 0 created or updated./) + ).to_stdout + end + + end + + context 'when dealing with pending services' do + let!(:service_pending_with_no_snapshot) { FactoryBot.create(:service, approved: false) } + let!(:service_pending_with_no_approved_snapshot) { FactoryBot.create(:service, approved: false) } + let!(:service_pending_with_approved_snapshot_but_not_public) { FactoryBot.create(:service, approved: false, visible: false, discarded_at: 1.day.ago) } + let!(:service_pending_with_approved_snapshot) { FactoryBot.create(:service, approved: false) } + + before do + # create another version but still don't approve it + service_pending_with_no_approved_snapshot.update(updated_at: Date.today) + + # approve a version, then change it so its pending again + service_pending_with_approved_snapshot.update(approved: true) + service_pending_with_approved_snapshot.update(updated_at: Date.today, approved: false) + + # approve a version, then change it so its pending again + service_pending_with_approved_snapshot_but_not_public.update(approved: true) + service_pending_with_approved_snapshot_but_not_public.update(updated_at: Date.today, approved: false) + end + + it 'doesn\'t add any that are unapproved' do + # no snapshot at all + allow(collection).to receive(:find_one_and_update).with( + { id: service_pending_with_no_snapshot.id }, + IndexedServicesSerializer.new(service_pending_with_no_snapshot).as_json, + { upsert: true } + ) + + # snapshot but not approved + allow(collection).to receive(:find_one_and_update).with( + { id: service_pending_with_no_approved_snapshot.id }, + IndexedServicesSerializer.new(service_pending_with_no_approved_snapshot).as_json, + { upsert: true } + ) + + # approved - it's approved but not visible so it wont be added + allow(collection).to receive(:find_one_and_update).with( + { id: service_pending_with_approved_snapshot_but_not_public.id }, + IndexedServicesSerializer.new(Service.from_hash(service_pending_with_approved_snapshot_but_not_public.last_approved_snapshot.object)).as_json, + { upsert: true } + ) + + # approved - it's approved, it should be added + allow(collection).to receive(:find_one_and_update).with( + { id: service_pending_with_approved_snapshot.id }, + IndexedServicesSerializer.new(Service.from_hash(service_pending_with_approved_snapshot.last_approved_snapshot.object)).as_json, + { upsert: true } + ) + + + indexed_ids = [service_pending_with_approved_snapshot.id] + allow(collection).to receive(:find).with({ id: { '$nin': indexed_ids.sort } }) + + expect { task.invoke }.to output( + match(/No unaccounted for services left in the index./) + .and match(/ 👉 1 updated or created/) + .and match(/ 👉 3 skipped/) + .and match(/ 👉 0 deleted/) + .and match(/ 👉 0 active services, 0 created or updated/) + .and match(/ 👉 0 temporarily_closed services, 0 created or updated/) + .and match(/ 👉 0 scheduled services, 0 created or updated/) + .and match(/ 👉 0 archived services, 0 deleted/) + .and match(/ 👉 0 expired services, 0 deleted/) + .and match(/ 👉 0 invisible services, 0 deleted/) + .and match(/ 👉 0 marked_for_deletion services, 0 deleted/) + .and match(/ 👉 4 pending services, 1 created or updated./) + ).to_stdout + end + + end + +end + +describe 'develop update_public_index task' do + + let(:task) { Rake::Task['update_public_index'] } + let(:collection) { double('indexed_services') } + + before do + Rake.application.rake_require('tasks/update_public_index') + Rake::Task.define_task(:environment) + + database = double('Database') + allow(database).to receive(:[]).with(:indexed_services).and_return(collection) + allow(Mongo::Client).to receive(:new).and_return(double('Mongo::Client', database: database)) + end + + after do + task.reenable # Re-enable the task after each test + end + + + let!(:service_active) { FactoryBot.create_list(:service, 3) } + let!(:service_temporarily_closed) { FactoryBot.create_list(:service, 3, temporarily_closed: true) } + let!(:service_scheduled) { FactoryBot.create_list(:service, 3, visible_from: Date.today + 2) } + + let!(:service_archived) { FactoryBot.create_list(:service, 3, discarded_at: 1.day.ago) } + let!(:service_expired) { FactoryBot.create_list(:service, 3, visible_to: Date.today - 2) } + let!(:service_invisible) { FactoryBot.create_list(:service, 3, visible: false) } + let!(:service_marked_for_deletion) { FactoryBot.create_list(:service, 3, marked_for_deletion: Time.now) } + + let!(:service_pending_with_no_snapshot) { FactoryBot.create(:service, approved: false) } + let!(:service_pending_with_no_approved_snapshot) { FactoryBot.create(:service, approved: false) } + let!(:service_pending_with_approved_snapshot_but_not_public) { FactoryBot.create(:service, approved: false, visible: false, discarded_at: 1.day.ago) } + let!(:service_pending_with_approved_snapshot) { FactoryBot.create(:service, approved: false) } + + before do + # create another version but still don't approve it + service_pending_with_no_approved_snapshot.update(updated_at: Date.today) + + # approve a version, then change it so its pending again + service_pending_with_approved_snapshot.update(approved: true) + service_pending_with_approved_snapshot.update(updated_at: Date.today, approved: false) + + # approve a version, then change it so its pending again + service_pending_with_approved_snapshot_but_not_public.update(approved: true) + service_pending_with_approved_snapshot_but_not_public.update(updated_at: Date.today, approved: false) + end + + it 'runs' do + skip("is used for testing purposes") + expect(collection).to receive(:find_one_and_update).at_least(:once) + expect(collection).to receive(:find_one_and_delete).at_least(:once) + expect(collection).to receive(:find).at_least(:once) + task.invoke + end + +end