Merge pull request #2518 from cloudfoundry/modern-uaa-integration-tests

Modern uaa integration tests

ci/dockerfiles/integration/Dockerfile

@@ -111,4 +111,8 @@ RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.
     libmysqlclient-dev \
     postgresql-client-${POSTGRES_MAJOR_VERSION}
 
-
+# Install UAA
+COPY bosh-deployment /usr/local/bosh-deployment/
+RUN \
+  UAA_RELEASE_URL="$(bosh int uaa.yml --path /release=uaa/value/url)" \
+  curl -o /usr/local/uaa.tgz "$UAA_RELEASE_URL"

ci/tasks/test-integration.sh

@@ -8,6 +8,8 @@ source bosh-src/ci/tasks/utils.sh
 check_param RUBY_VERSION
 check_param DB
 
+curl -o /usr/local/uaa.tgz "https://s3.amazonaws.com/bosh-compiled-release-tarballs/uaa-77.8.0-ubuntu-jammy-1.423-20240430-181143-092389865-20240430181145.tgz"
+
 echo "Starting $DB..."
 case "$DB" in
   mysql)
@@ -52,6 +54,7 @@ max_allowed_packet=6M' >> /etc/mysql/my.cnf
   postgresql)
     export PATH=/usr/lib/postgresql/$DB_VERSION/bin:$PATH
     export DB_PASSWORD="smurf"
+    export PGPASSWORD=${DB_PASSWORD}
 
     if [ ! -d /tmp/postgres ]; then # PostgreSQL hasn't been set up
       mkdir /tmp/postgres
@@ -96,7 +99,8 @@ max_allowed_packet=6M' >> /etc/mysql/my.cnf
         export PATH=/usr/lib/postgresql/$DB_VERSION/bin:$PATH
         export PGLOGS=/tmp/log/postgres
         export PGCLIENTENCODING=UTF8
-        pg_ctl start -l $PGLOGS/server.log -o "-N 400"
+        pg_ctl start -l $PGLOGS/server.log -o "-N 400" --wait
+        createdb -h 127.0.0.1 uaa
       '
     fi
     ;;

src/bosh-dev/assets/sandbox/config_server/certs/jwtVerification.key

@@ -1,6 +1,14 @@
 -----BEGIN PUBLIC KEY-----
-MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC/85WUjM7WKxnSMkd/TtGo+WWX
-kI5pH8ACBtb5oameiMJIw/wZ+dtyIwU+NZ0mKua+IzpGL1++bLJL0fDxE1LpD88j
-WzjaaTp8930UyY23oMxSa6JMG6OpDW469VTkY1g3uE/RAFtg7J9ZnDrTrMFJrduV
-VRs5sM3o4r1UYUTD6wIDAQAB
------END PUBLIC KEY-----
+MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA06hlEn4/NMWtnele3C5j
+PZm3P55/9jppIQXF/BP0OGaAMoRNZEafv6nH1b8btc4zs0dx52Y7xmOpeRqnXDeD
+3E2uTW0tgjsLQU8BbjlXRrKkzvUiYZqkaCkCMxcX5OcBvhT4Km+/1u4AGuFq9sS2
+Pdv+IAWEOsYbJjPl3lZn2uiQCCX5P5z5JkGbQ/7mBSi/ja1SogP1MBzRLf6VyHpE
+q7zvVnfm+oBsvPu2RC0EM14kL/TktQwyAvpL4TWunhE7gAh4j0fMNgmBfW9NG328
+E2+i+1ag+WyFzSy0rJQHgV0ImEtjhmj0E0C1ysI4Fpy5gew0ZrtsFwnXcKV39xMS
+EG3JLzV3h+QZ4BS9nBi/G8lLF3bWO/B0WTuYDWdkLm1ng3K/Oz0KhVkVG5Igu9FD
+k6EkD62SsMYUMl++3/EMrtNxxvJQDSOOf59/o3BQplbl6qOG6Mpji3ZggxyRYgRS
+iC7PPqJAKYrP3zCzeXyQEMMYxuOcmRR3W6aY341v+9Hs4w+zNJZ2DnB+r6Jaqhpi
+sSiu1yzT0nzkesdv47UNLaTkt0fFMbnvkUSvtz3ZRK5MqVAgjBMULqobz5ASPPe0
+RBg7V6023PHtyNxsJpxsobRG2aril4+7OOspiWSHIuoV1vm1IDs2utHak0GHY0Nc
+MBYj/GcsUXZhHBDbjHk07IsCAwEAAQ==
+-----END PUBLIC KEY-----

src/bosh-dev/assets/sandbox/director_test.yml.erb

@@ -7,7 +7,7 @@ runtime:
 
 port: <%= director_ruby_port %>
 
-mbus: nats://localhost:<%= nats_port %>
+mbus: "nats://localhost:<%= nats_port %>"
 
 logging:
   level: DEBUG
@@ -99,13 +99,23 @@ user_management:
   <% if user_authentication == 'uaa' %>
   uaa:
     symmetric_key: uaa-secret-key
-    # matches public key in asymmetric/uaa.yml
+    # public key of `uaa.jwt.policy.keys.key1.signingKey` from src/spec/assets/uaa_config/asymmetric/uaa.yml
+    # To Generate:
+    # openssl rsa -pubout -in <(yq .uaa.jwt.policy.keys.key1.signingKey src/spec/assets/uaa_config/asymmetric/uaa.yml)
     public_key: |
       -----BEGIN PUBLIC KEY-----
-      MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC/85WUjM7WKxnSMkd/TtGo+WWX
-      kI5pH8ACBtb5oameiMJIw/wZ+dtyIwU+NZ0mKua+IzpGL1++bLJL0fDxE1LpD88j
-      WzjaaTp8930UyY23oMxSa6JMG6OpDW469VTkY1g3uE/RAFtg7J9ZnDrTrMFJrduV
-      VRs5sM3o4r1UYUTD6wIDAQAB
+      MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA06hlEn4/NMWtnele3C5j
+      PZm3P55/9jppIQXF/BP0OGaAMoRNZEafv6nH1b8btc4zs0dx52Y7xmOpeRqnXDeD
+      3E2uTW0tgjsLQU8BbjlXRrKkzvUiYZqkaCkCMxcX5OcBvhT4Km+/1u4AGuFq9sS2
+      Pdv+IAWEOsYbJjPl3lZn2uiQCCX5P5z5JkGbQ/7mBSi/ja1SogP1MBzRLf6VyHpE
+      q7zvVnfm+oBsvPu2RC0EM14kL/TktQwyAvpL4TWunhE7gAh4j0fMNgmBfW9NG328
+      E2+i+1ag+WyFzSy0rJQHgV0ImEtjhmj0E0C1ysI4Fpy5gew0ZrtsFwnXcKV39xMS
+      EG3JLzV3h+QZ4BS9nBi/G8lLF3bWO/B0WTuYDWdkLm1ng3K/Oz0KhVkVG5Igu9FD
+      k6EkD62SsMYUMl++3/EMrtNxxvJQDSOOf59/o3BQplbl6qOG6Mpji3ZggxyRYgRS
+      iC7PPqJAKYrP3zCzeXyQEMMYxuOcmRR3W6aY341v+9Hs4w+zNJZ2DnB+r6Jaqhpi
+      sSiu1yzT0nzkesdv47UNLaTkt0fFMbnvkUSvtz3ZRK5MqVAgjBMULqobz5ASPPe0
+      RBg7V6023PHtyNxsJpxsobRG2aril4+7OOspiWSHIuoV1vm1IDs2utHak0GHY0Nc
+      MBYj/GcsUXZhHBDbjHk07IsCAwEAAQ==
       -----END PUBLIC KEY-----
     url: <%= uaa_url %>
   <% else %>
@@ -126,9 +136,9 @@ cpi:
   max_supported_api_version: 2
   preferred_api_version: <%= preferred_cpi_api_version %>
 
-<% if config_server_enabled %>
 config_server:
-  enabled: true
+  enabled: <%= config_server_enabled ? 'true' : 'false' %>
+<% if config_server_enabled %>
   url: <%= config_server_url %>
   ca_cert: |
     -----BEGIN CERTIFICATE-----
@@ -155,10 +165,6 @@ config_server:
     client_id: <%= config_server_uaa_client_id %>
     client_secret: <%= config_server_uaa_client_secret %>
     ca_cert_path: <%= config_server_uaa_ca_cert_path %>
-
-<% else %>
-config_server:
-  enabled: false
 <% end %>
 
 generate_vm_passwords: <%= generate_vm_passwords %>

src/bosh-dev/lib/bosh/dev/sandbox/director_config.rb

@@ -1,3 +1,5 @@
+require 'bosh/dev/sandbox/services/uaa_service'
+
 module Bosh::Dev::Sandbox
   class DirectorConfig
     attr_accessor :audit_log_path
@@ -69,7 +71,7 @@ def initialize(attrs, port_provider)
       @cloud_storage_dir = attrs.fetch(:cloud_storage_dir)
 
       @user_authentication = attrs.fetch(:user_authentication)
-      @uaa_url = "https://localhost:#{port_provider.get_port(:nginx)}/uaa"
+      @uaa_url = "https://127.0.0.1:8443"
 
       @config_server_enabled = attrs.fetch(:config_server_enabled)
       @config_server_url = "https://127.0.0.1:#{port_provider.get_port(:config_server_port)}"

src/bosh-dev/lib/bosh/dev/sandbox/main.rb

@@ -12,7 +12,6 @@
 require 'bosh/dev/sandbox/port_provider'
 require 'bosh/dev/sandbox/services/director_service'
 require 'bosh/dev/sandbox/services/nginx_service'
-require 'bosh/dev/sandbox/services/uaa_service'
 require 'bosh/dev/sandbox/services/config_server_service'
 require 'bosh/dev/gnatsd_manager'
 require 'cloud/dummy'
@@ -102,9 +101,8 @@ def initialize(db_opts, debug, test_env_number)
       @nats_log_path = File.join(@logs_path, 'nats.log')
       setup_nats
 
-      @uaa_service = UaaService.new(@port_provider, sandbox_root, base_log_path, @logger)
       @config_server_service = ConfigServerService.new(@port_provider, base_log_path, @logger, test_env_number)
-      @nginx_service = NginxService.new(sandbox_root, director_port, director_ruby_port, @uaa_service.port, base_log_path, @logger)
+      @nginx_service = NginxService.new(sandbox_root, director_port, director_ruby_port, "8443", base_log_path, @logger)
 
       @db_config = {
         ca_path: File.join(SANDBOX_ASSETS_DIR, 'database', 'rootCA.pem')
@@ -183,7 +181,6 @@ def start
         load_db_and_populate_blobstore(@test_initial_state)
       end
 
-      @uaa_service.start if @user_authentication == 'uaa'
       @config_server_service.start(@with_config_server_trusted_certs) if @config_server_enabled
 
       dir_config = director_config
@@ -268,7 +265,6 @@ def stop
       @nats_process.stop
 
       @health_monitor_process.stop
-      @uaa_service.stop
 
       @config_server_service.stop
 
@@ -473,7 +469,6 @@ def do_reset
 
       @director_service.start(director_config)
 
-      @uaa_service.start if @user_authentication == 'uaa'
       @nginx_service.restart_if_needed
 
       write_in_sandbox(EXTERNAL_CPI_CONFIG, load_config_template(EXTERNAL_CPI_CONFIG_TEMPLATE))

src/bosh-dev/lib/bosh/dev/sandbox/services/uaa_service.rb

@@ -1,40 +1,31 @@
-require 'common/retryable'
+require 'yaml'
+require 'json'
+require 'fileutils'
+require 'tmpdir'
+require 'bosh/template/evaluation_context'
+require 'bosh/dev/sandbox/service.rb'
+require 'erb'
 
 module Bosh::Dev::Sandbox
   class UaaService
     attr_reader :port
 
-    TOMCAT_VERSIONED_FILENAME = 'apache-tomcat-8.0.21'.freeze
-    UAA_FILENAME = 'uaa.war'.freeze
-
-    UAA_VERSION = 'cloudfoundry-identity-uaa-3.5.0'.freeze
-
+    COMPILED_UAA_RELEASE_PATH = '/usr/local/uaa.tgz'.freeze
+    UAA_BIN_PATH = '/var/vcap/jobs/uaa/bin/'.freeze
     REPO_ROOT = File.expand_path('../../../../../../', File.dirname(__FILE__))
-    INSTALL_DIR = File.join('tmp', 'integration-uaa', UAA_VERSION)
-    TOMCAT_DIR = File.join(INSTALL_DIR, TOMCAT_VERSIONED_FILENAME)
 
-    WAR_FILE_PATH = File.join(REPO_ROOT, TOMCAT_DIR, 'webapps', UAA_FILENAME)
     # Keys and Certs
     ASSETS_DIR = File.expand_path('bosh-dev/assets/sandbox/ca', REPO_ROOT)
     CERTS_DIR = File.expand_path('certs', ASSETS_DIR)
     ROOT_CERT = File.join(CERTS_DIR, 'rootCA.pem')
+    ROOT_KEY = File.join(CERTS_DIR, 'rootCA.key')
+    SERVER_CERT = File.join(CERTS_DIR, 'server.crt')
+    SERVER_KEY = File.join(CERTS_DIR, 'server.key')
 
-    def initialize(port_provider, sandbox_root, base_log_path, logger)
-      @port = port_provider.get_port(:uaa_http)
-      @server_port = port_provider.get_port(:uaa_server)
-
+    def initialize(sandbox_root, base_log_path, logger)
       @logger = logger
-      @build_mutex = Mutex.new
       @log_location = "#{base_log_path}.uaa.out"
 
-      @connector = HTTPEndpointConnector.new('uaa', 'localhost', @port, '/uaa/login', 'Reset password', @log_location, logger)
-
-      @uaa_webapps_path = File.join(sandbox_root, 'uaa.webapps')
-      unless File.exist? @uaa_webapps_path
-        FileUtils.mkdir_p @uaa_webapps_path
-        FileUtils.cp WAR_FILE_PATH, @uaa_webapps_path
-      end
-
       @config_path = File.join(sandbox_root, 'uaa_config')
       FileUtils.mkdir_p(@config_path)
       write_config_path
@@ -43,28 +34,76 @@ def initialize(port_provider, sandbox_root, base_log_path, logger)
     end
 
     def self.install
-      FileUtils.mkdir_p(TOMCAT_DIR)
-
-      retryable.retryer do
-        `#{File.dirname(__FILE__)}/install_tomcat.sh #{INSTALL_DIR} #{TOMCAT_VERSIONED_FILENAME} 957e88df8a9c3fc6b786321c4014b44c5c775773`
-        $? == 0
+      %w{
+        /var/vcap/sys/run/uaa
+        /var/vcap/sys/log/uaa
+        /var/vcap/data/tmp
+        /var/vcap/data/uaa
+        /var/vcap/data/uaa/cert-cache
+      }.each {|path| FileUtils.mkdir_p path}
+
+      installed_uaa_job_path = File.join('/', 'var', 'vcap', 'jobs', 'uaa')
+
+      Dir.mktmpdir do |workspace|
+        `tar xzf #{COMPILED_UAA_RELEASE_PATH} -C #{workspace}`
+        uaa_job_path = File.join(workspace, 'uaa')
+        FileUtils.mkdir_p uaa_job_path
+        `tar xzf #{File.join(workspace, 'jobs', 'uaa.tgz')} -C #{uaa_job_path}`
+        uaa_job_spec_path = File.join(uaa_job_path, 'job.MF')
+        job_spec = YAML.load_file(uaa_job_spec_path)
+        job_spec['packages'].each do |package_name|
+          package_path = File.join('/', 'var', 'vcap', 'packages', package_name)
+          FileUtils.mkdir_p(package_path)
+          `tar xzf #{File.join(workspace, 'compiled_packages', "#{package_name}.tgz")} -C #{package_path}`
+        end
+
+        context = {
+          'properties' => {
+            'uaa' => {
+              'sslCertificate' => File.read(SERVER_CERT),
+              'sslPrivateKey' => File.read(SERVER_KEY)
+            }
+          }
+        }
+
+        job_spec['properties'].map do |key, value|
+          next unless value.has_key?('default')
+          keys = key.split('.')
+          hash_segment =context['properties']
+          keys.each_with_index do |key, index|
+            if index == keys.length - 1
+              hash_segment[key] ||= value['default']
+            else
+              hash_segment[key] ||= {}
+            end
+            hash_segment = hash_segment[key]
+          end
+        end
+
+        context['properties'].deep_merge!(YAML.load_file(File.expand_path(File.join('spec','assets','uaa_config', 'asymmetric', 'uaa.yml'), REPO_ROOT)))
+        templates = job_spec['templates']
+        templates.each do |src, dst|
+          src_path = File.join(uaa_job_path, 'templates', src)
+          dest_path = File.join(installed_uaa_job_path, dst)
+          FileUtils.mkdir_p(File.dirname(dest_path))
+
+          evaluation_context = Bosh::Template::EvaluationContext.new(context, nil)
+          template = ERB.new(File.read(src_path), trim_mode: "-")
+          template_result = template.result(evaluation_context.get_binding)
+          File.write(dest_path, template_result)
+        end
       end
 
-      retryable.retryer do
-        `#{File.dirname(__FILE__)}/install_binary.sh #{UAA_VERSION}.war #{WAR_FILE_PATH} 6167d1b5afe3e12c26482fcb45c0056475cb3e1b9ca2996707d9ac9c22f60dc9 bosh-dependencies`
-        $? == 0
-      end
-    end
-
-    def self.retryable
-      Bosh::Retryable.new(tries: 6)
+      `chmod +x #{File.join(installed_uaa_job_path, 'bin', '*')}`
     end
 
     def start
+      system('useradd -ms /bin/bash vcap')
+      system(File.join(UAA_BIN_PATH, 'pre-start')) || raise
       @uaa_process.start
 
       begin
-        @connector.try_to_connect(6000)
+        system(File.join(UAA_BIN_PATH, 'post-start')) || raise
       rescue StandardError
         output_service_log(@uaa_process.description, @uaa_process.stdout_contents, @uaa_process.stderr_contents)
         raise
@@ -81,23 +120,24 @@ def stop
 
     def initialize_uaa_process
       opts = {
-        'uaa.http_port' => @port,
-        'uaa.server_port' => @server_port,
         'uaa.access_log_dir' => File.dirname(@log_location),
-        'uaa.webapps' => @uaa_webapps_path,
         'securerandom.source' => 'file:/dev/urandom',
       }
 
       catalina_opts = ' -Xms512M -Xmx512M '
       catalina_opts += opts.map { |key, value| "-D#{key}=#{value}" }.join(' ')
 
       Service.new(
-        [executable_path, 'run', '-config', server_xml],
+        [File.join(UAA_BIN_PATH, 'uaa')],
         {
           output: @log_location,
           env: {
             'CATALINA_OPTS' => catalina_opts,
-            'UAA_CONFIG_PATH' => @config_path,
+            'CATALINA_BASE' => '/var/vcap/data/uaa/tomcat',
+            'CATALINA_HOME' => '/var/vcap/data/uaa/tomcat',
+            'CLOUDFOUNDRY_CONFIG_PATH' => '/var/vcap/jobs/uaa/config',
+            'CLOUDFOUNDRY_LOG_PATH' => '/var/vcap/sys/log/uaa',
+            'JAVA_HOME' => ''
           },
         },
         @logger,
@@ -108,14 +148,6 @@ def working_dir
       File.expand_path('spec/assets/uaa', REPO_ROOT)
     end
 
-    def executable_path
-      File.join(TOMCAT_DIR, 'bin', 'catalina.sh')
-    end
-
-    def server_xml
-      File.join(REPO_ROOT, 'bosh-dev', 'assets', 'sandbox', 'tomcat-server.xml')
-    end
-
     def write_config_path
       spec_assets_base_path = 'spec/assets/uaa_config'
 

src/bosh-dev/lib/bosh/dev/sandbox/workspace.rb

@@ -1,4 +1,5 @@
 require 'bosh/dev'
+require 'fileutils'
 
 module Bosh::Dev::Sandbox
   class Workspace
@@ -29,6 +30,16 @@ def clean
         FileUtils.rm_rf(base_dir)
       end
 
+      def start_uaa
+        log_dir = File.join(dir, 'uaa_logs')
+        FileUtils.mkdir_p(log_dir)
+        uaa_log_file = File.open(File.join(log_dir, 'uaa_service.log'), 'w+')
+        logger = Logging.logger(uaa_log_file)
+        uaa_service = UaaService.new(File.join(dir, 'sandbox'), log_dir, logger)
+        uaa_service.start
+        uaa_service
+      end
+
       private
 
       def base_dir