diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 77701d7e2..b79162119 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -27,6 +27,7 @@ jobs:
           - '2.2'
           - '2.1'
         db: ['']
+        ssl: ['']
         include:
           # Comment out due to ci/setup.sh stucking.
           # - {os: ubuntu-18.04, ruby: 2.4, db: mariadb10.1}
@@ -39,22 +40,20 @@ jobs:
           # MariaDB lastet version
           # Allow failure due to the following test failures that rarely happens.
           # https://github.com/brianmario/mysql2/issues/1194
-          - {os: macos-latest, ruby: '2.6', db: mariadb, allow-failure: true}
+          - {os: macos-latest, ruby: '2.6', db: mariadb, ssl: openssl@1.1, allow-failure: true}
           # MySQL latest version
           # Allow failure due to the issue #1194.
-          - {os: macos-latest, ruby: '2.6', db: mysql, allow-failure: true}
+          - {os: macos-latest, ruby: '2.6', db: mysql, ssl: openssl@1.1, allow-failure: true}
       # On the fail-fast: true, it cancels all in-progress jobs
       # if any matrix job fails unlike Travis fast_finish.
       fail-fast: false
     env:
       BUNDLE_WITHOUT: development
+      # reduce MacOS CI time, don't need to clean a runtime that isn't saved
+      HOMEBREW_NO_INSTALL_CLEANUP: 1
+      HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1
     steps:
       - uses: actions/checkout@v3
-      - name: Install openssl
-        if: matrix.os == 'macos-latest'
-        run: |
-          brew update
-          brew install openssl
       # https://github.com/ruby/setup-ruby
       - uses: ruby/setup-ruby@v1
         with:
@@ -62,6 +61,8 @@ jobs:
           bundler-cache: true # runs 'bundle install' and caches installed gems automatically
       - if: matrix.db != ''
         run: echo 'DB=${{ matrix.db }}' >> $GITHUB_ENV
+      - if: startsWith(matrix.os, 'macos') && matrix.ssl != ''
+        run: echo "RUBY_MYSQL2_SSL_DIR=$(brew --prefix ${{ matrix.ssl }})" >> $GITHUB_ENV
       - run: sudo echo "127.0.0.1 mysql2gem.example.com" | sudo tee -a /etc/hosts
       - run: bash ci/setup.sh
       - run: bundle exec rake spec
diff --git a/ext/mysql2/extconf.rb b/ext/mysql2/extconf.rb
index a6417acf8..5fefa06aa 100644
--- a/ext/mysql2/extconf.rb
+++ b/ext/mysql2/extconf.rb
@@ -28,8 +28,16 @@ def add_ssl_defines(header)
 
 # Homebrew openssl
 if RUBY_PLATFORM =~ /darwin/ && system("command -v brew")
-  openssl_location = `brew --prefix openssl`.strip
-  $LDFLAGS << " -L#{openssl_location}/lib" if openssl_location
+  _, lib = dir_config('ssl')
+  unless lib
+    openssl_location = `brew --prefix openssl`.strip
+    lib = "#{openssl_location}/lib" if openssl_location
+  end
+
+  if lib
+    abort "-----\nCannot find library dir(s) #{lib}\n-----" unless lib && lib.split(File::PATH_SEPARATOR).any? { |dir| File.directory?(dir) }
+    $LDFLAGS << " -L#{lib}"
+  end
 end
 
 # 2.1+
diff --git a/tasks/compile.rake b/tasks/compile.rake
index 07aa1abea..fe30e7003 100644
--- a/tasks/compile.rake
+++ b/tasks/compile.rake
@@ -51,6 +51,9 @@ Rake::ExtensionTask.new("mysql2", Mysql2::GEMSPEC) do |ext|
       POST_INSTALL_MESSAGE
     end
   end
+
+  ssl_dir = ENV['RUBY_MYSQL2_SSL_DIR']
+  ext.config_options << "--with-ssl-dir=#{ssl_dir}" if ssl_dir
 end
 Rake::Task[:spec].prerequisites << :compile