allow rails to be loaded in fix auth

When objects are serialized into yaml blobs in tables,
we need to load our whole environment to handle the deserialization

This typically happens with miq_requests.options

- add --allow-failures flag
- add note where error was found
- require standard rails when allow failures
- continue despite errors when allow failures
- display status, counts, and # errors
- change order of fixes
- reduce columns brought back
- sending error output to STDERR
- only bring back request/task records that have v2 encoded passwords
- drop tests for v0 encoded passwords

spec/tools/fix_auth/auth_model_spec.rb

@@ -11,10 +11,8 @@
   let(:enc_v1)  { MiqPassword.new.encrypt(pass, "v1", v1_key) }
   let(:enc_v2)  { MiqPassword.new.encrypt(pass) }
   let(:bad_v2)  { "v2:{5555555555555555555555==}" }
-  let(:enc_leg) { v0_key.encrypt64(pass) }
 
   before do
-    MiqPassword.add_legacy_key(v0_key, :v0)
     MiqPassword.add_legacy_key(v1_key, :v1)
   end
 
@@ -24,13 +22,12 @@
 
   context "#authentications" do
     subject { FixAuth::FixAuthentication }
-    let(:contenders) { subject.contenders.collect(&:name) }
+    let(:contenders) { subject.contenders.select(:name).collect(&:name) }
     let(:v1_v2)  { subject.create(:name => "v2_v1", :password => enc_v2, :auth_key => enc_v1) }
     let(:v2_v1)  { subject.create(:name => "v1_v2", :password => enc_v1, :auth_key => enc_v2) }
     let(:v1)     { subject.create(:name => "v1", :password => enc_v1) }
     let(:v2)     { subject.create(:name => "v2", :password => enc_v2) }
     let(:badv2)  { subject.create(:name => "badv2", :password => bad_v2) }
-    let(:leg)    { subject.create(:name => "lg", :password => enc_leg) }
     let(:nls)    { subject.create(:name => "nls") }
     let(:not_c)  { subject.create(:name => "notc", :password => "nope") }
 
@@ -48,8 +45,7 @@
     end
 
     it "should build selection criteria (non selects)" do
-      expect(subject.selection_criteria).to match(/OR/)
-      expect(subject.selection_criteria).to match(/password.*<>.*''.*OR.*auth_key.*<>.*''/)
+      expect(subject.selection_criteria).to match(/password.*OR.*auth_key/)
     end
 
     it "should not find empty records" do
@@ -58,8 +54,8 @@
     end
 
     it "should find records with encrypted passwords" do
-      [v1, v2, leg, nls].each(&:save!)
-      expect(contenders).to include(v1.name, leg.name, v2.name)
+      [v2, nls].each(&:save!)
+      expect(contenders).to include(v2.name)
       expect(contenders).not_to include(nls.name)
     end
 
@@ -75,14 +71,6 @@
         expect(nls).not_to be_password_changed
       end
 
-      it "should upgrade legacy columns" do
-        subject.fix_passwords(leg)
-        expect(leg).to be_password_changed
-        expect(leg).not_to be_auth_key_changed
-        expect(leg.password).to be_encrypted(pass)
-        expect(leg.password).to be_encrypted_version(2)
-      end
-
       it "should upgrade v1 columns" do
         subject.fix_passwords(v1)
         expect(v1).to be_password_changed
@@ -105,13 +93,6 @@
     end
 
     context "#hardcode" do
-      it "should upgrade legacy columns" do
-        subject.fix_passwords(leg, :hardcode => "newpass")
-        expect(leg.password).to be_encrypted("newpass")
-        expect(leg.password).to be_encrypted_version(2)
-        expect(leg.auth_key).to be_blank
-      end
-
       it "should upgrade v2 columns" do
         subject.fix_passwords(v2, :hardcode => "newpass")
         expect(v2.password).to be_encrypted("newpass")
@@ -151,12 +132,12 @@
     subject { FixAuth::FixMiqAeValue }
 
     let(:pass_field) { FixAuth::FixMiqAeField.new(:name => "pass", :datatype => "password") }
-    let(:v1) { subject.create(:field => pass_field, :value => enc_v1) }
+    let(:v2) { subject.create(:field => pass_field, :value => enc_v2) }
 
     it "should update with complex contenders" do
-      v1 # make sure record exists
+      v2 # make sure record exists
       subject.run(:silent => true)
-      expect(v1.reload.value).to be_encrypted_version(2)
+      expect(v2.reload.value).to be_encrypted_version(2)
     end
   end
 

tools/fix_auth/auth_config_model.rb

@@ -42,8 +42,10 @@ def recrypt(old_value, options = {})
         symbol_keys ? hash.deep_symbolize_keys! : hash.deep_stringify_keys!
         hash.to_yaml
       rescue ArgumentError # undefined class/module
-        puts "potentially bad yaml:"
-        puts old_value
+        unless options[:allow_failures]
+          STDERR.puts "potentially bad yaml:"
+          STDERR.puts old_value
+        end
         raise
       end
     end

tools/fix_auth/auth_model.rb

@@ -11,14 +11,18 @@ def available_columns
         column_names & password_columns
       end
 
+      def select_columns
+        [:id] + available_columns
+      end
+
       def contenders
-        where(selection_criteria)
+        where(selection_criteria).select(select_columns)
       end
 
-      # bring back anything with a password column that is not nil or blank
+      # bring back anything with a password column that has a non blank v1 or v2 password in it
       def selection_criteria
         available_columns.collect do |column|
-          "(COALESCE(#{column},'') <> '')"
+          "(#{column} like '%v2:{%')"
         end.join(" OR ")
       end
 
@@ -81,16 +85,32 @@ def display_column(r, column, options)
       def run(options = {})
         return if available_columns.empty?
         puts "fixing #{table_name}.#{available_columns.join(", ")}" unless options[:silent]
+        processed = 0
+        errors = 0
         contenders.each do |r|
-          fix_passwords(r, options)
-          if options[:verbose]
-            display_record(r)
-            available_columns.each do |column|
-              display_column(r, column, options)
+          begin
+            fix_passwords(r, options)
+            if options[:verbose]
+              display_record(r)
+              available_columns.each do |column|
+                display_column(r, column, options)
+              end
             end
+            r.save! if !options[:dry_run] && r.changed?
+            processed += 1
+          rescue ArgumentError # undefined class/module
+            errors += 1
+            unless options[:allow_failures]
+              STDERR.puts "unable to fix #{r.class.table_name}:#{r.id}" unless options[:silent]
+              raise
+            end
+          end
+          if !options[:silent] && (errors + processed) % 10_000 == 0
+            puts "processed #{processed} with #{errors} errors"
           end
-          r.save! unless options[:dry_run]
         end
+        puts "#{options[:dry_run] ? "viewed" : "processed"} #{processed} records" unless options[:silent]
+        puts "found #{errors} errors" if errors > 0 && !options[:silent]
       end
 
       def clean_up

tools/fix_auth/cli.rb

@@ -25,6 +25,7 @@ def parse(args, env = {})
         opt :databaseyml, "Rewrite database.yml", :type => :boolean, :short => "y", :default => false
         opt :db,       "Upgrade database",  :type => :boolean, :short => 'x', :default => false
         opt :legacy_key, "Legacy Key",      :type => :string, :short => "K"
+        opt :allow_failures, "Run through all records, even with errors", :type => :boolean, :short => nil, :default => false
       end
 
       options[:database] = args.first || "vmdb_production"

tools/fix_auth/fix_auth.rb

@@ -26,7 +26,7 @@ def db_attributes(database)
     end
 
     def run_options
-      options.slice(:verbose, :dry_run, :hardcode, :invalid)
+      options.slice(:verbose, :dry_run, :hardcode, :invalid, :allow_failures)
     end
 
     def database
@@ -35,7 +35,7 @@ def database
 
     def models
       [FixAuthentication, FixMiqDatabase, FixMiqAeValue, FixMiqAeField,
-       FixMiqRequest, FixMiqRequestTask, FixSettingsChange]
+       FixSettingsChange, FixMiqRequest, FixMiqRequestTask]
     end
 
     def generate_password
@@ -69,6 +69,10 @@ def fix_database_yml
       FixDatabaseYml.run({:hardcode => options[:password]}.merge(run_options))
     end
 
+    def load_rails
+      require File.expand_path("../../../config/application.rb", __FILE__)
+    end
+
     def set_passwords
       MiqPassword.key_root = cert_dir if cert_dir
       MiqPassword.add_legacy_key("v0_key", :v0)
@@ -83,6 +87,7 @@ def run
 
       generate_password if options[:key]
       fix_database_yml if options[:databaseyml]
+      load_rails if options[:allow_failures]
       fix_database_passwords if options[:db]
     end
   end

tools/fix_auth/models.rb

@@ -56,10 +56,6 @@ class FixMiqRequest < ActiveRecord::Base
     self.password_prefix = "password::"
     self.symbol_keys = true
     self.table_name = "miq_requests"
-
-    def self.contenders
-      where("options like '%password%'")
-    end
   end
 
   class FixMiqRequestTask < ActiveRecord::Base
@@ -71,10 +67,6 @@ class FixMiqRequestTask < ActiveRecord::Base
     self.password_prefix = "password::"
     self.symbol_keys = true
     self.table_name = "miq_request_tasks"
-
-    def self.contenders
-      where("options like '%password%'")
-    end
   end
 
   class FixSettingsChange < ActiveRecord::Base
@@ -113,6 +105,10 @@ def load
       self
     end
 
+    def changed?
+      true
+    end
+
     def save!
       File.write(id, @yaml)
     end