Make run_sqitch work services beyond erchef

Partybus.run_sqitch was previously hard-coded
to apply database migrations only to opscode_chef,
from the opscode_chef schema path. It also
re-applied the legacy 1.0.4 schema with every new schema change.

This change updates the function so that it only applies the migration
it's told to apply. It also expands it to permit other services
to invoke run_sqitch for updating their own databases.

The partybus migrations were retrofitted to use the new form.

In addition migration 014 was updated to explicitly apply the
legacy 1.0.4 schema.

This was added as a precaution -- there should be no path that
has migration 14 (or later) running, that does not already have
the baseline sqitch deployments applied:
 * new installations will already have the baseline properly applied.
 * EC 11.current will already have the baseline
   properly applied, as sqitch was introduced sometime early in the
   11.x timeframe.
 * Upgrading from 11.old (pre-sqitch) requires an upgrade to
   11.x current before proceeding.

omnibus/files/private-chef-upgrades/001/014_upgrade_migration_schema.rb

@@ -1,10 +1,18 @@
 define_upgrade do
 
   if Partybus.config.bootstrap_server
-
     must_be_data_master
 
-    # run 2.2.4 migration which includes schema upgrade for migration state
-    run_sqitch("@2.2.4", "@1.0.4")
+    # This ensures that sqitch is properly set up, even when upgrading from private chef (which did
+    # not use sqitch), as well as ensuring the baseline schema is present for opscode_chef.
+    #
+    # Note that we only have to apply it in this upgrade, because this is the first sqitch migration
+    # that will be run.  If this doesn't need to be run, that tells us they're upgrading from an installation
+    # that already has applied it.
+    run_sqitch('@1.0.4', 'oc_erchef', path: 'oc_erchef/schema/baseline')
+
+    # The actual schema change for this release -
+    # track more state values in migration_state.
+    run_sqitch('@2.2.4', 'oc_erchef')
   end
 end

omnibus/files/private-chef-upgrades/001/016_add_org_tables_osc_hash_types.rb

@@ -1,11 +1,10 @@
 define_upgrade do
 
   if Partybus.config.bootstrap_server
-
     must_be_data_master
 
-    # run 2.4.0 migrations to update db - includes adding org association/user tables
+    # schema updates include adding org association/user tables
     # and adding the OSC password hash types to the password_hash_type_enum
-    run_sqitch("@2.4.0", "@1.0.4")
+    run_sqitch('@2.4.0', 'oc_erchef')
   end
 end

omnibus/files/private-chef-upgrades/001/020_multi_key_schema_migration.rb

@@ -1,12 +1,14 @@
 define_upgrade do
   require "pg"
-  # sql user info will be in one of two places - under 'postgresql' or under 'opscode-erchef' in more
-  # recent versions. If someone is upgrading from an earlier versions, it may not yet
-  # have been moved to its new location.
   if Partybus.config.bootstrap_server
     must_be_data_master
-    run_sqitch("@2.5.3", "@1.0.4")
+    # Multi-key support
+    run_sqitch('@2.5.3', 'oc_erchef')
 
+    # Migrate key data to new schema:
+    # sql user info will be in one of two places - under 'postgresql' or under 'opscode-erchef' in more
+    # recent versions. If someone is upgrading from an earlier versions, it may not yet
+    # have been moved to its new location.
     running_config = JSON.parse(File.read("/etc/opscode/chef-server-running.json"))
     pc = running_config['private_chef']
     keyname = pc['opscode-erchef'].has_key?('sql_user') ? 'opscode-erchef' : 'postgresql'

omnibus/files/private-chef-upgrades/001/021_key_schema_migration_2.rb

@@ -1,6 +1,8 @@
 define_upgrade do
   if Partybus.config.bootstrap_server
     must_be_data_master
-    run_sqitch("@2.9.0", "@1.0.4")
+    # 1. cookbook artifacts
+    # 2. adds last update tracking to keys table.
+    run_sqitch('@2.9.0', 'oc_erchef')
   end
 end

omnibus/files/private-chef-upgrades/001/022_cbv_type_addition.rb

@@ -1,6 +1,7 @@
 define_upgrade do
   if Partybus.config.bootstrap_server
     must_be_data_master
-    run_sqitch("@cbv-type", "@1.0.4")
+    # Performance improvements for cookbook fetching.
+    run_sqitch('@cbv-type', 'oc_erchef')
   end
 end

omnibus/files/private-chef-upgrades/001/027_node_policyfile_fields.rb

@@ -1,7 +1,8 @@
 define_upgrade do
   if Partybus.config.bootstrap_server
     must_be_data_master
-    run_sqitch("@node-policyfile-fields", "@1.0.4")
+    # Add policyfile fields to node table
+    run_sqitch('@node-policyfile-fields', 'oc_erchef')
   end
 end
 

omnibus/partybus/lib/partybus.rb

@@ -10,6 +10,9 @@ def self.configure(&block)
 
   class Config
 
+    SECRETS_FILE = "/etc/opscode/private-chef-secrets.json"
+    RUNNING_CONFIG_FILE = "/etc/opscode/chef-server-running.json"
+
     attr_accessor :database_connection_string
     attr_accessor :database_unix_user
     attr_accessor :database_migration_directory
@@ -23,20 +26,34 @@ class Config
 
     attr_accessor :running_server
     attr_accessor :postgres
+    attr_accessor :secrets
 
     def initialize
-      if File.exists?("/etc/opscode/chef-server-running.json")
-        @running_server = JSON.parse(IO.read("/etc/opscode/chef-server-running.json"))
+      if File.exists?(RUNNING_CONFIG_FILE)
+        @running_server = JSON.parse(IO.read(RUNNING_CONFIG_FILE))
         @postgres = @running_server['private_chef']['postgresql']
       else
         log <<EOF
 ***
-ERROR: Cannot find /etc/opscode/chef-server-running.json
+ERROR: Cannot find #{RUNNING_CONFIG_FILE}
+***
+Try running `chef-server-ctl reconfigure` before upgrading.
+
+EOF
+        exit(1)
+      end
+      if File.exists?(SECRETS_FILE)
+        @secrets = JSON.parse(IO.read(SECRETS_FILE))
+      else
+        log <<EOF
+***
+ERROR: Cannot find #{SECRETS_FILE}
 ***
-Try running `chef-server-ctl reconfigure` before upgrading
+Try running `chef-server-ctl reconfigure` before upgrading.
 
 EOF
         exit(1)
+
       end
     end
 

omnibus/partybus/lib/partybus/migration_api/v1.rb

@@ -97,19 +97,38 @@ def run_command(command, options={})
         runner.run_command(command, options)
       end
 
-      def run_sqitch(ec_target, osc_target="@1.0.4")
-        args =  " --db-user #{Partybus.config.postgres['db_superuser']}"
-        args << " --db-name opscode_chef"
-        args << " --db-host #{Partybus.config.postgres['vip']}"
-        args << " --db-port #{Partybus.config.postgres['port']}"
-        args << " --engine pg"
-
-        cmd = "cd /opt/opscode/embedded/service/opscode-erchef/schema"
-        cmd << " && cd baseline && sqitch #{args} deploy --to-target #{osc_target} --verify"
-        cmd << " && cd .. && sqitch #{args} deploy --to-target #{ec_target} --verify;"
-        run_command(cmd,
-                    :cwd => "/opt/opscode/embedded/service/opscode-erchef/schema",
-                    :env => {"PGPASSWORD" => Partybus.config.postgres['db_superuser_password']})
+      # Caller may use 'opts' to override :username :password,
+      # :database, and/or :path.
+      def run_sqitch(target, service, opts = {})
+        options = default_opts_for_service(service).merge(opts)
+        command = <<-EOM.gsub(/\s+/," ").strip!
+          sqitch --engine pg
+            --db-name #{options[:database]}
+            --db-host #{Partybus.config.postgres['vip']}
+            --db-port #{Partybus.config.postgres['port']}
+            --db-user #{options[:username]}
+            --top-dir /opt/opscode/embedded/service/#{options[:path]}
+            deploy #{target} --verify
+        EOM
+        run_command(command, env: {"PGPASSWORD" => options[:password]})
+      end
+
+      def default_opts_for_service(service)
+        username = Partybus.config.postgres['db_superuser']
+        password = Partybus.config.postgres['db_superuser_password']
+        path = "#{service}/schema"
+        case service
+        when 'oc_erchef'
+          database = 'opscode_chef'
+        when 'oc_bifrost'
+          database = 'bifrost'
+          path = 'oc_bifrost/db'
+        when 'bookshelf'
+          database = 'bookshelf'
+        end
+
+        {username: username, password: password,
+         database: database, path: path }
       end
 
 

src/bookshelf/schema/README.md

@@ -10,6 +10,9 @@ installed/reconfigured.  In order to have a schema change applied,
 you must add a partybus migration with content similar to the following:
 
     define_upgrade do
-      run_sqitch(target: "target-tag-name", database: "bookshelf")
+      if Partybus.config.bootstrap_server
+        must_be_data_master
+        run_sqitch("target-tag-name", 'bookshelf')
+                   'bookshelf', service: 'bookshelf')
     end
 

src/oc_bifrost/schema/README.md

@@ -14,11 +14,12 @@ installed/reconfigured.  In order to have a schema change applied,
 you must add a partybus migration with content similar to the following:
 
     define_upgrade do
-      run_sqitch(target: "target-tag-name", database: "bifrost")
+      if Partybus.config.bootstrap_server
+        must_be_data_master
+        run_sqitch("target-tag-name", 'oc_bifrost')
     end
 
 
-
 # Testing
 
 We use [pgTAP][] to test both the schema and the stored procedures in