Feature: Implement CI deployment for development servers (#251)

* add test and appliance environment to deploy action

* add PRIVATE_CONFIG_REPO environment for the deploy action

* revert removing the jumphost step in deploy action

* set default deploy branch for each of the environments

* set the deployment rails_env to "appliance" for all the environments

* use a variable for the config folder path to use in the get_config task

* extract deploy ssh_options into the deploy.rb file

* add test environment capistrano deploy file

* make the jumpbox  host and username configurable for the deployment

* make PRIVATE_CONFIG_REPO default example use https with github PAT token

* remove the on push in development branch auto deploy

.github/workflows/deploy.yml

@@ -9,6 +9,7 @@
 # GH_PAT - github Personal Access Token for accessing private config repo
 #
 # SSH_JUMPHOST - ssh jump/proxy host though which deployments have to though if UI nodes live on private network.
+# SSH_JUMPHOST_USER - username  to use to connect to the ssh jump/proxy.
 #
 # DEPLOY_ENC_KEY - key for decrypting deploymnet ssh key residing in config/
 # this SSH key is used for accessing jump host, UI nodes, and private github repo.
@@ -17,26 +18,27 @@ name: Capistrano Deployment
 # Controls when the action will run.
 on:
   push:
-    branches: [ development staging ]
+    branches: [ stage test]
   # Allows running this workflow manually from the Actions tab
   workflow_dispatch:
     inputs:
       BRANCH:
         description: 'Branch/tag to deploy'
         options:
-          - development
-          - staging
+          - stage
+          - test
           - master
-        default: staging
+        default: test
         required: true
       environment:
         description: 'target environment to deploy to'
         type: choice
         options:
-          - development
           - staging
           - production
-        default: staging
+          - appliance
+          - test
+        default: test
 jobs:
   deploy:
     runs-on: ubuntu-latest
@@ -55,6 +57,13 @@ jobs:
 
         USER_INPUT_ENVIRONMENT=${{ inputs.environment }}
         echo "TARGET=${USER_INPUT_ENVIRONMENT:-staging}" >> $GITHUB_ENV
+        
+        CONFIG_REPO=${{ secrets.CONFIG_REPO }}
+        GH_PAT=${{ secrets.GH_PAT }}
+        echo "PRIVATE_CONFIG_REPO=https://${GH_PAT}@github.com/${CONFIG_REPO}" >> $GITHUB_ENV
+        
+        echo "SSH_JUMPHOST=${{ secrets.SSH_JUMPHOST }}" >> $GITHUB_ENV
+        echo "SSH_JUMPHOST_USER=${{ secrets.SSH_JUMPHOST_USER }}" >> $GITHUB_ENV
     # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
     - uses: actions/checkout@v3
     - uses: ruby/setup-ruby@v1
@@ -69,6 +78,13 @@ jobs:
         path:  deploy_config
     - name: copy-deployment-config
       run:  cp -r deploy_config/ontoportal_web_ui/${{ inputs.environment }}/* .
+      # add ssh hostkey so that capistrano doesn't complain
+    - name: Add jumphost's hostkey to Known Hosts
+      run: |
+        mkdir -p ~/.ssh
+        echo "${{ secrets.SSH_JUMPHOST }}"
+        ssh-keyscan -H ${{ secrets.SSH_JUMPHOST }} > ~/.ssh/known_hosts
+      shell: bash
     - uses: miloserdow/capistrano-deploy@master
       with:
         target: ${{ env.TARGET }} # which environment to deploy

config/deploy.rb

@@ -11,7 +11,6 @@
 # default deployment branch is master which can be overwritten with BRANCH env var
 # BRANCH env var can be set to specific branch of tag, i.e 'v6.8.1'
 
-set :branch, ENV.include?('BRANCH') ? ENV['BRANCH'] : 'master'
 
 # Default deploy_to directory is /var/www/my_app
 set :deploy_to, "/srv/ontoportal/#{fetch(:application)}"
@@ -42,7 +41,8 @@
 set :keep_releases, 5
 set :bundle_without, 'development:test'
 set :bundle_config, { deployment: true }
-
+set :rails_env, "appliance"
+set :config_folder_path, "#{fetch(:application)}/#{fetch(:stage)}"
 # Defaults to [:web]
 set :assets_roles, [:web, :app]
 set :keep_assets, 3
@@ -55,6 +55,43 @@
 # If you don't set `:passenger_restart_with_touch`, capistrano-passenger will check what version of passenger you are running
 # and use `passenger-config restart-app` if it is available in that version.
 
+# you can set custom ssh options
+# it's possible to pass any option but you need to keep in mind that net/ssh understand limited list of options
+# you can see them in [net/ssh documentation](http://net-ssh.github.io/net-ssh/classes/Net/SSH.html#method-c-start)
+# set it globally
+#  set :ssh_options, {
+#    keys: %w(/home/rlisowski/.ssh/id_rsa),
+#    forward_agent: false,
+#    auth_methods: %w(password)
+#  }
+# and/or per server
+# server 'example.com',
+#   user: 'user_name',
+#   roles: %w{web app},
+#   ssh_options: {
+#     user: 'user_name', # overrides user setting above
+#     keys: %w(/home/user_name/.ssh/id_rsa),
+#     forward_agent: false,
+#     auth_methods: %w(publickey password)
+#     # password: 'please use keys'
+#   }
+# setting per server overrides global ssh_options
+
+SSH_JUMPHOST = ENV.include?('SSH_JUMPHOST') ? ENV['SSH_JUMPHOST'] : 'jumpbox.hostname.com'
+SSH_JUMPHOST_USER = ENV.include?('SSH_JUMPHOST_USER') ? ENV['SSH_JUMPHOST_USER'] : 'username'
+
+JUMPBOX_PROXY = "#{SSH_JUMPHOST_USER}@#{SSH_JUMPHOST}"
+set :ssh_options, {
+  user: 'ontoportal',
+  forward_agent: 'true',
+  keys: %w(config/deploy_id_rsa),
+  auth_methods: %w(publickey),
+  # use ssh proxy if UI servers are on a private network
+  proxy: Net::SSH::Proxy::Command.new("ssh #{JUMPBOX_PROXY} -W %h:%p")
+}
+
+#private git repo for configuraiton
+PRIVATE_CONFIG_REPO = ENV.include?('PRIVATE_CONFIG_REPO') ? ENV['PRIVATE_CONFIG_REPO'] : 'https://[email protected]/your_organization/ontoportal-configs.git'
 desc "Check if agent forwarding is working"
 task :forwarding do
   on roles(:all) do |h|
@@ -83,7 +120,7 @@
       TMP_CONFIG_PATH = "/tmp/#{SecureRandom.hex(15)}".freeze
       on roles(:app) do
         execute "git clone -q #{PRIVATE_CONFIG_REPO} #{TMP_CONFIG_PATH}"
-        execute "rsync -a #{TMP_CONFIG_PATH}/#{fetch(:application)}/ #{release_path}/"
+        execute "rsync -a #{TMP_CONFIG_PATH}/#{fetch(:config_folder_path)}/ #{release_path}/"
         execute "rm -rf #{TMP_CONFIG_PATH}"
       end
     elsif defined?(LOCAL_CONFIG_PATH)
@@ -102,6 +139,7 @@
   end
 
 
+  after :updating, :get_config
   after :publishing, :restart
 
   after :restart, :clear_cache do

config/deploy/production.rb

@@ -6,7 +6,7 @@
 # Don't declare `role :all`, it's a meta role
 role :app, %w{ui1.prd.ontoportal.org ui2.prd.ontoportal.org}
 role :db, %w{ui1.prd.ontoportal.org} # sufficient to run db:migrate only on one system
-
+set :branch, ENV.include?('BRANCH') ? ENV['BRANCH'] : 'master'
 # Extended Server Syntax
 # ======================
 # This can be used to drop a more detailed server
@@ -15,35 +15,3 @@
 # extended properties on the server.
 #server 'example.com', user: 'deploy', roles: %w{web app}, my_property: :my_value
 set :log_level, :error
-# you can set custom ssh options
-# it's possible to pass any option but you need to keep in mind that net/ssh understand limited list of options
-# you can see them in [net/ssh documentation](http://net-ssh.github.io/net-ssh/classes/Net/SSH.html#method-c-start)
-# set it globally
-#  set :ssh_options, {
-#    keys: %w(/home/rlisowski/.ssh/id_rsa),
-#    forward_agent: false,
-#    auth_methods: %w(password)
-#  }
-# and/or per server
-# server 'example.com',
-#   user: 'user_name',
-#   roles: %w{web app},
-#   ssh_options: {
-#     user: 'user_name', # overrides user setting above
-#     keys: %w(/home/user_name/.ssh/id_rsa),
-#     forward_agent: false,
-#     auth_methods: %w(publickey password)
-#     # password: 'please use keys'
-#   }
-# setting per server overrides global ssh_options
-set :ssh_options, {
-  user: 'deployer',
-  forward_agent: 'true',
-  keys: %w(config/deploy_id_rsa),
-  auth_methods: %w(publickey),
-  # use ssh proxy if UI servers are on a private network
-  proxy: Net::SSH::Proxy::Command.new('ssh [email protected] -W %h:%p')
-}
-
-#private git repo for configuraiton
-PRIVATE_CONFIG_REPO = ENV.include?('PRIVATE_CONFIG_REPO') ? ENV['PRIVATE_CONFIG_REPO'] : '[email protected]:author/private_config_repo.git'

config/deploy/staging.rb

@@ -6,7 +6,7 @@
 # Don't declare `role :all`, it's a meta role
 role :app, %w{stageportal.lirmm.fr}
 role :db, %w{stageportal.lirmm.fr} # sufficient to run db:migrate only on one system
-
+set :branch, ENV.include?('BRANCH') ? ENV['BRANCH'] : 'stage'
 # Extended Server Syntax
 # ======================
 # This can be used to drop a more detailed server
@@ -15,35 +15,3 @@
 # extended properties on the server.
 #server 'example.com', user: 'deploy', roles: %w{web app}, my_property: :my_value
 set :log_level, :error
-# you can set custom ssh options
-# it's possible to pass any option but you need to keep in mind that net/ssh understand limited list of options
-# you can see them in [net/ssh documentation](http://net-ssh.github.io/net-ssh/classes/Net/SSH.html#method-c-start)
-# set it globally
-#  set :ssh_options, {
-#    keys: %w(/home/rlisowski/.ssh/id_rsa),
-#    forward_agent: false,
-#    auth_methods: %w(password)
-#  }
-# and/or per server
-# server 'example.com',
-#   user: 'user_name',
-#   roles: %w{web app},
-#   ssh_options: {
-#     user: 'user_name', # overrides user setting above
-#     keys: %w(/home/user_name/.ssh/id_rsa),
-#     forward_agent: false,
-#     auth_methods: %w(publickey password)
-#     # password: 'please use keys'
-#   }
-# setting per server overrides global ssh_options
-set :ssh_options, {
-  user: 'deploy',
-  forward_agent: 'true',
-  keys: %w(config/deploy_id_rsa),
-  auth_methods: %w(publickey),
-  # use ssh proxy if UI servers are on a private network
-  # proxy: Net::SSH::Proxy::Command.new('ssh [email protected] -W %h:%p')
-}
-
-#private git repo for configuraiton
-PRIVATE_CONFIG_REPO = ENV.include?('PRIVATE_CONFIG_REPO') ? ENV['PRIVATE_CONFIG_REPO'] : '[email protected]:author/private_config_repo.git'

config/deploy/test.rb

@@ -0,0 +1,17 @@
+# Simple Role Syntax
+# ==================
+# Supports bulk-adding hosts to roles, the primary
+# server in each group is considered to be the first
+# unless any hosts have the primary property set.
+# Don't declare `role :all`, it's a meta role
+role :app, %w{testportal.lirmm.fr}
+role :db, %w{testportal.lirmm.fr} # sufficient to run db:migrate only on one system
+# Extended Server Syntax
+# ======================
+# This can be used to drop a more detailed server
+# definition into the server list. The second argument
+# something that quacks like a hash can be used to set
+# extended properties on the server.
+#server 'example.com', user: 'deploy', roles: %w{web app}, my_property: :my_value
+set :log_level, :error
+set :branch, ENV.include?('BRANCH') ? ENV['BRANCH'] : 'test'