diff --git a/lib/tasks/build_public_index.rake b/lib/tasks/build_public_index.rake index 2e27d59a..e3a624ef 100644 --- a/lib/tasks/build_public_index.rake +++ b/lib/tasks/build_public_index.rake @@ -1,4 +1,4 @@ -desc 'Build public index in the Mongo API' +desc 'Wipe and rebuild public index in the Mongo API' task :build_public_index => :environment do # Turn off logging for this rake task, otherwise it just fills up our logs dev_null = Logger.new('/dev/null') diff --git a/lib/tasks/check_public_index.rake b/lib/tasks/check_public_index.rake new file mode 100644 index 00000000..a2da495c --- /dev/null +++ b/lib/tasks/check_public_index.rake @@ -0,0 +1,103 @@ +desc 'Compare Outpost and public index in the Mongo API' +task :check_public_index => :environment do + # Turn off logging for this rake task, otherwise it just fills up our logs + dev_null = Logger.new('/dev/null') + Rails.logger = dev_null + ActiveRecord::Base.logger = dev_null + + Mongo::Logger.logger.level = Logger::FATAL + + puts "⏰ Connecting to mongo database..." + client = Mongo::Client.new(ENV["DB_URI"] || 'mongodb://root:password@localhost:27017/outpost_development?authSource=admin', { + retry_writes: false + }) + collection = client.database[:indexed_services] + + + active_services, temporarily_closed_services, scheduled_services, archived_services, expired_services, invisible_services, marked_for_deletion_services, pending_services = [], [], [], [], [], [], [], [] + + + mongo_ids = collection.find.map { |doc| doc[:id] } + + Service.find_each do |service| + case service.status + when 'active' + if !mongo_ids.include?(service.id) + active_services << service + end + when 'temporarily closed' + if !mongo_ids.include?(service.id) + temporarily_closed_services << service + end + when 'scheduled' + if !mongo_ids.include?(service.id) + scheduled_services << service + end + when 'archived' + if mongo_ids.include?(service.id) + archived_services << service + end + when 'expired' + if mongo_ids.include?(service.id) + expired_services << service + end + when 'invisible' + if mongo_ids.include?(service.id) + invisible_services << service + end + when 'marked for deletion' + if mongo_ids.include?(service.id) + marked_for_deletion_services << service + end + when 'pending' + approved_alternative = service.last_approved_snapshot + if approved_alternative && approved_alternative.object['visible'] == true && approved_alternative.object['discarded_at'].blank? + if !mongo_ids.include?(service.id) + pending_services << service + end + end + end + end + + + + + puts "Active services not in index: #{active_services.count}" + active_services.each do |service| + puts "#{service.id} - #{service.name} - #{service.status} - #{service.updated_at}" + end + puts "Temporarily closed services not in index: #{temporarily_closed_services.count}" + temporarily_closed_services.each do |service| + puts "#{service.id} - #{service.name} - #{service.status} - #{service.updated_at}" + end + puts "Scheduled services not in index: #{scheduled_services.count}" + scheduled_services.each do |service| + puts "#{service.id} - #{service.name} - #{service.status} - #{service.updated_at}" + end + puts "Archived services in index: #{archived_services.count}" + archived_services.each do |service| + puts "#{service.id} - #{service.name} - #{service.status} - #{service.updated_at}" + end + puts "Expired services in index: #{expired_services.count}" + expired_services.each do |service| + puts "#{service.id} - #{service.name} - #{service.status} - #{service.updated_at}" + end + puts "Invisible services in index: #{invisible_services.count}" + invisible_services.each do |service| + puts "#{service.id} - #{service.name} - #{service.status} - #{service.updated_at}" + end + puts "Marked for deletion services in index: #{marked_for_deletion_services.count}" + marked_for_deletion_services.each do |service| + puts "#{service.id} - #{service.name} - #{service.status} - #{service.updated_at}" + end + puts "Pending services not in index: #{pending_services.count}" + pending_services.each do |service| + puts "#{service.id} - #{service.name} - #{service.status} - #{service.updated_at}" + end + + +end + + +def create_update_service(service) +end \ No newline at end of file diff --git a/lib/tasks/update_public_index.rake b/lib/tasks/update_public_index.rake new file mode 100644 index 00000000..178ae92b --- /dev/null +++ b/lib/tasks/update_public_index.rake @@ -0,0 +1,162 @@ +desc 'Update Outpost public index copies in the Mongo API' +task :update_public_index => :environment do + # Turn off logging for this rake task, otherwise it just fills up our logs + dev_null = Logger.new('/dev/null') + Rails.logger = dev_null + ActiveRecord::Base.logger = dev_null + + Mongo::Logger.logger.level = Logger::FATAL + + puts "⏰ Connecting to mongo database..." + client = Mongo::Client.new(ENV["DB_URI"] || 'mongodb://root:password@localhost:27017/outpost_development?authSource=admin', { + retry_writes: false + }) + collection = client.database[:indexed_services] + approved_count, unapproved_count, deleted_count, deleted_skipped_count = 0, 0, 0, 0 + active_count, temporarily_closed_count, scheduled_count, archived_count, expired_count, invisible_count, marked_for_deletion_count, pending_count = 0, 0, 0, 0, 0, 0, 0, 0 + active_ids, temporarily_closed_ids, scheduled_ids, archived_ids, expired_ids, invisible_ids, marked_for_deletion_ids, pending_ids = [], [], [], [], [], [], [], [] + + Service.find_each do |service| + case service.status + + # active services are all indexed + when 'active' + upsert_active = collection.find_one_and_update({ id: service.id }, + IndexedServicesSerializer.new(service).as_json, + { upsert: true }) + active_count += 1 + active_ids << upsert_active[:id] + puts "ACTIVE: #{service.name} indexed" + + # temporarily closed services are all indexed + when 'temporarily closed' + upsert_temporarily_closed = collection.find_one_and_update({ id: service.id }, + IndexedServicesSerializer.new(service).as_json, + { upsert: true }) + temporarily_closed_count += 1 + temporarily_closed_ids << upsert_temporarily_closed[:id] + puts "TEMPORARILY CLOSED: #{service.name} indexed" + + # scheduled services are all indexed - the API determines if they're returned or not + when 'scheduled' + upsert_scheduled = collection.find_one_and_update({ id: service.id }, + IndexedServicesSerializer.new(service).as_json, + { upsert: true }) + scheduled_count += 1 + scheduled_ids << upsert_scheduled[:id] + puts "SCHEDULED: #{service.name} indexed" + + + # archived services are removed from the index + when 'archived' + deleted_archived = collection.find_one_and_delete({ id: service.id }) + archived_count += 1 + if deleted_archived + puts "🗑 ARCHIVED: #{service.name} deleted" + archived_ids << deleted_archived[:id] + else + puts "⚠️ ARCHIVED: #{service.name} not found in index, skipping" + end + + # expired services are removed from the index + when 'expired' + deleted_expired = collection.find_one_and_delete({ id: service.id }) + expired_count += 1 + if deleted_expired + puts "🗑 EXPIRED: #{service.name} deleted" + expired_ids << deleted_expired[:id] + else + puts "⚠️ EXPIRED: #{service.name} not found in index, skipping" + end + + # invisible services are removed from the index + when 'invisible' + deleted_invisible = collection.find_one_and_delete({ id: service.id }) + deleted_count += 1 + if deleted_invisible + puts "🗑 INVISIBLE: #{service.name} deleted" + invisible_ids << deleted_invisible[:id] + else + puts "⚠️ INVISIBLE: #{service.name} not found in index, skipping" + end + + # marked for deletion definitely get removed from the index + when 'marked for deletion' + deleted_marked_for_deletion = collection.find_one_and_delete({ id: service.id }) + marked_for_deletion_count += 1 + if deleted_marked_for_deletion + puts "🗑 MARKED FOR DELETION: #{service.name} deleted" + marked_for_deletion_ids << deleted_marked_for_deletion[:id] + else + puts "⚠️ MARKED FOR DELETION: #{service.name} not found in index, skipping" + end + + # if status is pending we work off last approved snapshot, if it exists and is visible or we make a snapshot to make it visible + when 'pending' + approved_alternative = service.last_approved_snapshot + unless approved_alternative + puts "🚨 No alternative approved snapshot of #{service.name} exists. Skipping." + next + end + + unless approved_alternative.object['visible'] == true && approved_alternative.object['discarded_at'].blank? + puts "🚨 Approved snapshot of #{service.name} is not publicly visible. Skipping." + next + end + + snapshot = Service.from_hash(approved_alternative.object) + upsert_pending = collection.find_one_and_update({ id: service.id }, + IndexedServicesSerializer.new(snapshot).as_json, + { upsert: true }) + puts "🤔 Alternative approved snapshot of #{service.name} indexed" + pending_count += 1 + pending_ids << upsert_pending[:id] + end + end + + + + # check if we missed any entries + indexed_ids = active_ids + temporarily_closed_ids + scheduled_ids + missed_services = collection.find({ id: { '$nin': indexed_ids } }) + + if missed_services.count > 0 + puts "\n\n" + puts "#{missed_services.count} missed services, deleting..." + missed_service_ids = missed_services.map { |service| service['id'] } + delete_missed_services = collection.delete_many({ id: { '$in': missed_service_ids } }) + delete_missed_services.each do |service| + puts "🗑 #{service.name} deleted" + end + puts "#{delete_missed_services.deleted_count} missed services deleted." + end + + deleted_ids = archived_ids + expired_ids + invisible_ids + marked_for_deletion_ids + + + puts "\n\n" + puts "🏁🏁 SUMMARY 🏁🏁" + puts " 👉 #{indexed_ids.length + pending_ids.length} updated or created" + puts " 👉 #{deleted_ids.length} deleted" + + puts "\n\n\n" + puts "Updated or Created Services" + puts " 👉 #{active_count} active services, #{active_ids.length} created or updated" + puts " 👉 #{temporarily_closed_count} temporarily_closed services, #{temporarily_closed_ids.length} created or updated" + puts " 👉 #{scheduled_count} scheduled services, #{scheduled_ids.length} created or updated" + + puts "\n\n\n" + puts "Deleted Services" + puts " 👉 #{archived_count} archived services, #{archived_ids.length} deleted" + puts " 👉 #{expired_count} expired services, #{expired_ids.length} deleted" + puts " 👉 #{invisible_count} invisible services, #{invisible_ids.length} deleted" + puts " 👉 #{marked_for_deletion_count} marked_for_deletion services, #{marked_for_deletion_ids.length} deleted" + + puts "\n\n\n" + puts "Pending Services" + puts " 👉 #{pending_count} pending services, #{pending_ids.length} created." +end + + +def create_update_service(service) +end \ No newline at end of file