bifrost and bookshelf schemas explicit upgrade

omnibus/files/private-chef-cookbooks/private-chef/recipes/bifrost_database.rb

@@ -19,6 +19,9 @@
 
 private_chef_pg_database 'bifrost' do
   owner bifrost_attrs['sql_user']
+  # This is used to trigger creation of the schema during install.
+  # For upgrades, create a partybus migration to perform any schema changes.
+  notifies :deploy, "private_chef_pg_sqitch[/opt/opscode/embedded/service/oc_bifrost/db]", :immediately
 end
 
 private_chef_pg_user_table_access bifrost_attrs['sql_user'] do
@@ -33,10 +36,14 @@
   access_profile :read
 end
 
+# Note that these migrations are only deployed during an initial install via the
+# :deploy notification above.  Upgrades to existing installations must be managed
+# via partybus migrations.
 private_chef_pg_sqitch "/opt/opscode/embedded/service/oc_bifrost/db" do
   hostname postgres_attrs['vip']
   port     postgres_attrs['port']
   username  postgres_attrs['db_superuser']
   password  postgres_attrs['db_superuser_password']
   database "bifrost"
+  action :nothing
 end

omnibus/files/private-chef-cookbooks/private-chef/recipes/bookshelf_database.rb

@@ -14,6 +14,9 @@
 
 private_chef_pg_database 'bookshelf' do
   owner bookshelf_attrs['sql_user']
+  # This is used to trigger creation of the schema during install.
+  # For upgrades, create a partybus migration to perform any schema changes.
+  notifies :deploy, "private_chef_pg_sqitch[/opt/opscode/embedded/service/bookshelf/schema]", :immediately
 end
 
 private_chef_pg_user_table_access bookshelf_attrs['sql_user'] do
@@ -28,10 +31,14 @@
   access_profile :read
 end
 
+# Note that these migrations are only deployed during an initial install via the
+# :deploy notification above.  Upgrades to existing installations must be managed
+# via partybus migrations.
 private_chef_pg_sqitch "/opt/opscode/embedded/service/bookshelf/schema" do
   hostname postgres_attrs['vip']
   port     postgres_attrs['port']
   username  postgres_attrs['db_superuser']
   password  postgres_attrs['db_superuser_password']
   database "bookshelf"
+  action :nothing
 end

omnibus/files/private-chef-upgrades/001/014_upgrade_migration_schema.rb

@@ -1,10 +1,18 @@
 define_upgrade do
 
   if Partybus.config.bootstrap_server
-
     must_be_data_master
 
-    # run 2.2.4 migration which includes schema upgrade for migration state
-    run_sqitch("@2.2.4", "@1.0.4")
+    # This ensures that sqitch is properly set up, even when upgrading from private chef (which did
+    # not use sqitch), as well as ensuring the baseline schema is present for opscode_chef.
+    #
+    # Note that we only have to apply it in this upgrade, because this is the first sqitch migration
+    # that will be run.  If this doesn't need to be run, that tells us they're upgrading from an installation
+    # that already has applied it.
+    run_sqitch('@1.0.4', 'oc_erchef', path: 'opscode-erchef/schema/baseline')
+
+    # The actual schema change for this release -
+    # track more state values in migration_state.
+    run_sqitch('@2.2.4', 'oc_erchef')
   end
 end

omnibus/files/private-chef-upgrades/001/016_add_org_tables_osc_hash_types.rb

@@ -1,11 +1,10 @@
 define_upgrade do
 
   if Partybus.config.bootstrap_server
-
     must_be_data_master
 
-    # run 2.4.0 migrations to update db - includes adding org association/user tables
+    # schema updates include adding org association/user tables
     # and adding the OSC password hash types to the password_hash_type_enum
-    run_sqitch("@2.4.0", "@1.0.4")
+    run_sqitch('@2.4.0', 'oc_erchef')
   end
 end

omnibus/files/private-chef-upgrades/001/020_multi_key_schema_migration.rb

@@ -1,12 +1,14 @@
 define_upgrade do
   require "pg"
-  # sql user info will be in one of two places - under 'postgresql' or under 'opscode-erchef' in more
-  # recent versions. If someone is upgrading from an earlier versions, it may not yet
-  # have been moved to its new location.
   if Partybus.config.bootstrap_server
     must_be_data_master
-    run_sqitch("@2.5.3", "@1.0.4")
+    # Multi-key support
+    run_sqitch('@2.5.3', 'oc_erchef')
 
+    # Migrate key data to new schema:
+    # sql user info will be in one of two places - under 'postgresql' or under 'opscode-erchef' in more
+    # recent versions. If someone is upgrading from an earlier versions, it may not yet
+    # have been moved to its new location.
     running_config = JSON.parse(File.read("/etc/opscode/chef-server-running.json"))
     pc = running_config['private_chef']
     keyname = pc['opscode-erchef'].has_key?('sql_user') ? 'opscode-erchef' : 'postgresql'

omnibus/files/private-chef-upgrades/001/021_key_schema_migration_2.rb

@@ -1,6 +1,8 @@
 define_upgrade do
   if Partybus.config.bootstrap_server
     must_be_data_master
-    run_sqitch("@2.9.0", "@1.0.4")
+    # 1. cookbook artifacts
+    # 2. adds last update tracking to keys table.
+    run_sqitch('@2.9.0', 'oc_erchef')
   end
 end

omnibus/files/private-chef-upgrades/001/022_cbv_type_addition.rb

@@ -1,6 +1,7 @@
 define_upgrade do
   if Partybus.config.bootstrap_server
     must_be_data_master
-    run_sqitch("@cbv-type", "@1.0.4")
+    # Performance improvements for cookbook fetching.
+    run_sqitch('@cbv-type', 'oc_erchef')
   end
 end

omnibus/files/private-chef-upgrades/001/027_node_policyfile_fields.rb

@@ -1,7 +1,8 @@
 define_upgrade do
   if Partybus.config.bootstrap_server
     must_be_data_master
-    run_sqitch("@node-policyfile-fields", "@1.0.4")
+    # Add policyfile fields to node table
+    run_sqitch('@node-policyfile-fields', 'oc_erchef')
   end
 end
 

omnibus/partybus/lib/partybus.rb

@@ -10,6 +10,9 @@ def self.configure(&block)
 
   class Config
 
+    SECRETS_FILE = "/etc/opscode/private-chef-secrets.json"
+    RUNNING_CONFIG_FILE = "/etc/opscode/chef-server-running.json"
+
     attr_accessor :database_connection_string
     attr_accessor :database_unix_user
     attr_accessor :database_migration_directory
@@ -23,20 +26,34 @@ class Config
 
     attr_accessor :running_server
     attr_accessor :postgres
+    attr_accessor :secrets
 
     def initialize
-      if File.exists?("/etc/opscode/chef-server-running.json")
-        @running_server = JSON.parse(IO.read("/etc/opscode/chef-server-running.json"))
+      if File.exists?(RUNNING_CONFIG_FILE)
+        @running_server = JSON.parse(IO.read(RUNNING_CONFIG_FILE))
         @postgres = @running_server['private_chef']['postgresql']
       else
         log <<EOF
 ***
-ERROR: Cannot find /etc/opscode/chef-server-running.json
+ERROR: Cannot find #{RUNNING_CONFIG_FILE}
+***
+Try running `chef-server-ctl reconfigure` before upgrading.
+
+EOF
+        exit(1)
+      end
+      if File.exists?(SECRETS_FILE)
+        @secrets = JSON.parse(IO.read(SECRETS_FILE))
+      else
+        log <<EOF
+***
+ERROR: Cannot find #{SECRETS_FILE}
 ***
-Try running `chef-server-ctl reconfigure` before upgrading
+Try running `chef-server-ctl reconfigure` before upgrading.
 
 EOF
         exit(1)
+
       end
     end
 

omnibus/partybus/lib/partybus/migration_api/v1.rb

@@ -97,19 +97,39 @@ def run_command(command, options={})
         runner.run_command(command, options)
       end
 
-      def run_sqitch(ec_target, osc_target="@1.0.4")
-        args =  " --db-user #{Partybus.config.postgres['db_superuser']}"
-        args << " --db-name opscode_chef"
-        args << " --db-host #{Partybus.config.postgres['vip']}"
-        args << " --db-port #{Partybus.config.postgres['port']}"
-        args << " --engine pg"
-
-        cmd = "cd /opt/opscode/embedded/service/opscode-erchef/schema"
-        cmd << " && cd baseline && sqitch #{args} deploy --to-target #{osc_target} --verify"
-        cmd << " && cd .. && sqitch #{args} deploy --to-target #{ec_target} --verify;"
-        run_command(cmd,
-                    :cwd => "/opt/opscode/embedded/service/opscode-erchef/schema",
-                    :env => {"PGPASSWORD" => Partybus.config.postgres['db_superuser_password']})
+      # Caller may use 'opts' to override :username :password,
+      # :database, and/or :path.
+      def run_sqitch(target, service, opts = {})
+        options = default_opts_for_service(service).merge(opts)
+        command = <<-EOM.gsub(/\s+/," ").strip!
+          sqitch --engine pg
+            --db-name #{options[:database]}
+            --db-host #{Partybus.config.postgres['vip']}
+            --db-port #{Partybus.config.postgres['port']}
+            --db-user #{options[:username]}
+            --top-dir /opt/opscode/embedded/service/#{options[:path]}
+            deploy #{target} --verify
+        EOM
+        run_command(command, env: {"PGPASSWORD" => options[:password]})
+      end
+
+      def default_opts_for_service(service)
+        username = Partybus.config.postgres['db_superuser']
+        password = Partybus.config.postgres['db_superuser_password']
+        path = "#{service}/schema"
+        case service
+        when 'oc_erchef'
+          database = 'opscode_chef'
+          path = 'opscode-erchef/schema'
+        when 'oc_bifrost'
+          database = 'bifrost'
+          path = 'oc_bifrost/db'
+        when 'bookshelf'
+          database = 'bookshelf'
+        end
+
+        {username: username, password: password,
+         database: database, path: path }
       end
 
 

src/bookshelf/schema/README.md

@@ -0,0 +1,18 @@
+## Schema Install Note
+
+The complete schema is deployed as part of the chef-server installation
+via the `bookshelf_database` recipe.
+
+## Schema Upgrade Note
+
+Upgrades are not automatically applied when chef-server upgrades are
+installed/reconfigured.  In order to have a schema change applied,
+you must add a partybus migration with content similar to the following:
+
+    define_upgrade do
+      if Partybus.config.bootstrap_server
+        must_be_data_master
+        run_sqitch("target-tag-name", 'bookshelf')
+                   'bookshelf', service: 'bookshelf')
+    end
+

src/oc_bifrost/schema/README.md

@@ -7,6 +7,19 @@ Bifrost PostgreSQL Schema
 
 * [Set up and Use Sqitch](doc/sqitch_background.md)
 
+## Ensuring Modifications are Applied in Upgrades
+
+Upgrades are not automatically applied when chef-server upgrades are
+installed/reconfigured.  In order to have a schema change applied,
+you must add a partybus migration with content similar to the following:
+
+    define_upgrade do
+      if Partybus.config.bootstrap_server
+        must_be_data_master
+        run_sqitch("target-tag-name", 'oc_bifrost')
+    end
+
+
 # Testing
 
 We use [pgTAP][] to test both the schema and the stored procedures in