Merge pull request #939 from newrelic/55fromsource

CI: Ubuntu 20.04
newrelic · Feb 11, 2022 · 7460ced · 7460ced
2 parents f08dd65 + 8d61ce8
commit 7460ced
Show file tree

Hide file tree

Showing 4 changed files with 119 additions and 82 deletions.
diff --git a/.github/actions/build-ruby/dist/index.js b/.github/actions/build-ruby/dist/index.js
@@ -78,6 +78,12 @@ function usesOldOpenSsl(rubyVersion) {
   return rubyVersion.match(/^2\.[0123]/);
 }
 
+// all Rubies (v2.3..v2.5) testing Rails <= 4.x will use mysql2 0.3.x,
+// which will need an older MySQL
+function usesMySQL55(rubyVersion) {
+  return rubyVersion.match(/^2\.[012345]/)
+}
+
 // ruby-build is used to compile Ruby and it's various executables
 // rbenv's ruby-build doesn't fully support EOL rubies (< 2.4 at time of this writing).
 // setup-ruby also doesn't correctly build the older rubies with openssl support.
@@ -136,15 +142,11 @@ async function setupRubyEnvironment(rubyVersion) {
 // Sets up any options at the bundler level so that when gems that
 // need specific settings are installed, their specific flags are relayed.
 async function configureBundleOptions(rubyVersion) {
-  if (!usesOldOpenSsl(rubyVersion)) { return }
-
-  const openSslPath = rubyOpenSslPath(rubyVersion);
+  if (!usesMySQL55(rubyVersion)) { return }
 
-  // https://stackoverflow.com/questions/30834421/error-when-trying-to-install-app-with-mysql2-gem
   await exec.exec('bundle', [
     'config', '--global', 'build.mysql2',
-      `"--with-ldflags=-L${openSslPath}/lib"`,
-      `"--with-cppflags=-I${openSslPath}/include"`
+    '--with-mysql-config=/usr/local/mysql55/bin/mysql_config'
   ]);
 }
 
@@ -160,39 +162,38 @@ function prependEnv(envName, envValue, divider=' ') {
 // The older Rubies also need older MySQL that was built against the older OpenSSL libraries.
 // Otherwise mysql adapter will segfault in Ruby because it attempts to dynamically link
 // to the 1.1 series while Ruby links against the 1.0 series.
-async function downgradeMySQL() {
-  core.startGroup(`Downgrade MySQL`)
+async function installMySQL55(rubyVersion) {
+  if (!usesMySQL55(rubyVersion)) { return }
 
-  const pkgDir = `${process.env.HOME}/packages`
-  const pkgOption = `--directory-prefix=${pkgDir}/`
-  const mirrorUrl = 'https://security.debian.org/debian-security/pool/updates/main/m/mysql-5.5/'
-  const ubuntuUrl = 'http://archive.ubuntu.com/ubuntu/pool/main'
+  core.startGroup(`Install MySQL 5.5 for Older Rubies`)
 
-  // executes the following all in parallel
-  const promise1 = exec.exec('sudo', ['apt-get', 'remove', 'mysql-client'])
-  const promise2 = exec.exec('wget', [pkgOption, `${mirrorUrl}/libmysqlclient18_5.5.62-0%2Bdeb8u1_amd64.deb`])
-  const promise3 = exec.exec('wget', [pkgOption, `${mirrorUrl}/libmysqlclient-dev_5.5.62-0%2Bdeb8u1_amd64.deb`])
-  const promise4 = exec.exec('wget', [pkgOption, `${ubuntuUrl}/g/glibc/multiarch-support_2.27-3ubuntu1.2_amd64.deb`])
+  const filePath = '/usr/local/mysql55'
+  const key = 'v2-mysql55-cache'
 
-  // wait for the parallel processes to finish
-  await Promise.all([promise1, promise2, promise3, promise4])
+  let cachedKey = null
+  try {
+    cachedKey = await cache.restoreCache([filePath], key, [key])
+  } catch(error) {
+    if (error.name === cache.ValidationError.name) {
+      throw error;
+    } else {
+      core.info(`[warning] There was an error restoring the MySQL 5.5 cache ${error.message}`)
+    }
+  }
 
-  // executes serially
-  await exec.exec('sudo', ['dpkg', '-i', `${pkgDir}/multiarch-support_2.27-3ubuntu1.2_amd64.deb`])
-  await exec.exec('sudo', ['dpkg', '-i', `${pkgDir}/libmysqlclient18_5.5.62-0+deb8u1_amd64.deb`])
-  await exec.exec('sudo', ['dpkg', '-i', `${pkgDir}/libmysqlclient-dev_5.5.62-0+deb8u1_amd64.deb`])
+  if (cachedKey == null) {
+    await exec.exec('sudo', [`${process.env.GITHUB_WORKSPACE}/test/script/install_mysql55`])
+    try {
+      await cache.saveCache([filePath], key)
+    }
+    catch (error) {
+      console.log('Failed to save cache' + error.toString())
+    }
+  }
 
   core.endGroup()
 }
 
-// mySQL (and others) must be downgraded for EOL rubies for native extension
-// gems to install correctly and against the right openSSL libraries.
-async function downgradeSystemPackages(rubyVersion) {
-  if (!usesOldOpenSsl(rubyVersion)) { return }
-
-  await downgradeMySQL();
-}
-
 // any settings needed in all Ruby environments from EOL'd rubies to current
 async function setupAllRubyEnvironments() {
   core.startGroup("Setup for all Ruby Environments")
@@ -379,7 +380,6 @@ async function saveRubyToCache(rubyVersion) {
 
 // Ensures working, properly configured environment for running test suites.
 async function postBuildSetup(rubyVersion) {
-  await downgradeSystemPackages(rubyVersion)
   await setupRubyEnvironmentAfterBuild(rubyVersion)
   await configureBundleOptions(rubyVersion)
   await showVersions()
@@ -393,6 +393,7 @@ async function setupEnvironment(rubyVersion, dependencyList) {
   await installDependencies('workflow', dependencyList)
   await setupRubyEnvironment(rubyVersion)
   await addRubyToPath(rubyVersion)
+  await installMySQL55(rubyVersion)
 }
 
 async function setupRuby(rubyVersion){
@@ -57075,4 +57076,4 @@ module.exports = require("zlib");;
 /******/ 	// Load entry module and return exports
 /******/ 	return __nccwpck_require__(2932);
 /******/ })()
-;
+;
diff --git a/.github/actions/build-ruby/index.js b/.github/actions/build-ruby/index.js
@@ -71,6 +71,12 @@ function usesOldOpenSsl(rubyVersion) {
   return rubyVersion.match(/^2\.[0123]/);
 }
 
+// all Rubies (v2.3..v2.5) testing Rails <= 4.x will use mysql2 0.3.x,
+// which will need an older MySQL
+function usesMySQL55(rubyVersion) {
+  return rubyVersion.match(/^2\.[012345]/)
+}
+
 // ruby-build is used to compile Ruby and it's various executables
 // rbenv's ruby-build doesn't fully support EOL rubies (< 2.4 at time of this writing).
 // setup-ruby also doesn't correctly build the older rubies with openssl support.
@@ -129,15 +135,11 @@ async function setupRubyEnvironment(rubyVersion) {
 // Sets up any options at the bundler level so that when gems that
 // need specific settings are installed, their specific flags are relayed.
 async function configureBundleOptions(rubyVersion) {
-  if (!usesOldOpenSsl(rubyVersion)) { return }
-
-  const openSslPath = rubyOpenSslPath(rubyVersion);
+  if (!usesMySQL55(rubyVersion)) { return }
 
-  // https://stackoverflow.com/questions/30834421/error-when-trying-to-install-app-with-mysql2-gem
   await exec.exec('bundle', [
     'config', '--global', 'build.mysql2',
-      `"--with-ldflags=-L${openSslPath}/lib"`,
-      `"--with-cppflags=-I${openSslPath}/include"`
+    '--with-mysql-config=/usr/local/mysql55/bin/mysql_config'
   ]);
 }
 
@@ -153,39 +155,38 @@ function prependEnv(envName, envValue, divider=' ') {
 // The older Rubies also need older MySQL that was built against the older OpenSSL libraries.
 // Otherwise mysql adapter will segfault in Ruby because it attempts to dynamically link
 // to the 1.1 series while Ruby links against the 1.0 series.
-async function downgradeMySQL() {
-  core.startGroup(`Downgrade MySQL`)
+async function installMySQL55(rubyVersion) {
+  if (!usesMySQL55(rubyVersion)) { return }
 
-  const pkgDir = `${process.env.HOME}/packages`
-  const pkgOption = `--directory-prefix=${pkgDir}/`
-  const mirrorUrl = 'https://security.debian.org/debian-security/pool/updates/main/m/mysql-5.5/'
-  const ubuntuUrl = 'http://archive.ubuntu.com/ubuntu/pool/main'
+  core.startGroup(`Install MySQL 5.5 for Older Rubies`)
 
-  // executes the following all in parallel
-  const promise1 = exec.exec('sudo', ['apt-get', 'remove', 'mysql-client'])
-  const promise2 = exec.exec('wget', [pkgOption, `${mirrorUrl}/libmysqlclient18_5.5.62-0%2Bdeb8u1_amd64.deb`])
-  const promise3 = exec.exec('wget', [pkgOption, `${mirrorUrl}/libmysqlclient-dev_5.5.62-0%2Bdeb8u1_amd64.deb`])
-  const promise4 = exec.exec('wget', [pkgOption, `${ubuntuUrl}/g/glibc/multiarch-support_2.27-3ubuntu1.2_amd64.deb`])
+  const filePath = '/usr/local/mysql55'
+  const key = 'v2-mysql55-cache'
 
-  // wait for the parallel processes to finish
-  await Promise.all([promise1, promise2, promise3, promise4])
+  let cachedKey = null
+  try {
+    cachedKey = await cache.restoreCache([filePath], key, [key])
+  } catch(error) {
+    if (error.name === cache.ValidationError.name) {
+      throw error;
+    } else {
+      core.info(`[warning] There was an error restoring the MySQL 5.5 cache ${error.message}`)
+    }
+  }
 
-  // executes serially
-  await exec.exec('sudo', ['dpkg', '-i', `${pkgDir}/multiarch-support_2.27-3ubuntu1.2_amd64.deb`])
-  await exec.exec('sudo', ['dpkg', '-i', `${pkgDir}/libmysqlclient18_5.5.62-0+deb8u1_amd64.deb`])
-  await exec.exec('sudo', ['dpkg', '-i', `${pkgDir}/libmysqlclient-dev_5.5.62-0+deb8u1_amd64.deb`])
+  if (cachedKey == null) {
+    await exec.exec('sudo', [`${process.env.GITHUB_WORKSPACE}/test/script/install_mysql55`])
+    try {
+      await cache.saveCache([filePath], key)
+    }
+    catch (error) {
+      console.log('Failed to save cache' + error.toString())
+    }
+  }
 
   core.endGroup()
 }
 
-// mySQL (and others) must be downgraded for EOL rubies for native extension
-// gems to install correctly and against the right openSSL libraries.
-async function downgradeSystemPackages(rubyVersion) {
-  if (!usesOldOpenSsl(rubyVersion)) { return }
-
-  await downgradeMySQL();
-}
-
 // any settings needed in all Ruby environments from EOL'd rubies to current
 async function setupAllRubyEnvironments() {
   core.startGroup("Setup for all Ruby Environments")
@@ -372,7 +373,6 @@ async function saveRubyToCache(rubyVersion) {
 
 // Ensures working, properly configured environment for running test suites.
 async function postBuildSetup(rubyVersion) {
-  await downgradeSystemPackages(rubyVersion)
   await setupRubyEnvironmentAfterBuild(rubyVersion)
   await configureBundleOptions(rubyVersion)
   await showVersions()
@@ -386,6 +386,7 @@ async function setupEnvironment(rubyVersion, dependencyList) {
   await installDependencies('workflow', dependencyList)
   await setupRubyEnvironment(rubyVersion)
   await addRubyToPath(rubyVersion)
+  await installMySQL55(rubyVersion)
 }
 
 async function setupRuby(rubyVersion){

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -11,7 +11,7 @@ on:
 
 jobs:
   build-ruby:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         ruby-version: [2.2.10, 2.3.8, 2.4.10, 2.5.9, 2.6.9, 2.7.5, 3.0.3, 3.1.0, jruby-9.2.19.0]
@@ -25,7 +25,15 @@ jobs:
 
   unit-tests:
     needs: build-ruby
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
+    services:
+      mysql:
+        image: mysql:5.7
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: yes
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
+        ports:
+          - "3306:3306"
     strategy:
       fail-fast: false
       matrix:
@@ -74,23 +82,26 @@ jobs:
               }
             }
 
-      - name: Start mysql
-        run: sudo systemctl start mysql
-
       - name: Run Unit Tests
         uses: nick-invision/[email protected]
         with:
           timeout_minutes: 20
           max_attempts: 2
           command:  bundle exec rake test:env[${{ env.rails }}] TESTOPTS="--verbose"
         env:
-          DB_PORT: 3306
-          MYSQL_PASSWORD: root
+          DB_PORT: ${{ job.services.mysql.ports[3306] }}
 
   multiverse:
     needs: build-ruby
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     services:
+      mysql:
+        image: mysql:5.7
+        env:
+          MYSQL_ROOT_PASSWORD: root
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
+        ports:
+          - 3306
       postgres:
         image: postgres:latest
         env:
@@ -149,8 +160,8 @@ jobs:
         with:
           ruby-version: ${{ matrix.ruby-version }}
 
-      - name: Start mysql
-        run: sudo systemctl start mysql
+      - name: Test MySQL
+        run: mysql --host 127.0.0.1 --port ${{ job.services.mysql.ports[3306] }} -uroot -proot -e "SHOW GRANTS FOR 'root'@'localhost'"
 
       - name: Run Multiverse Tests
         uses: nick-invision/[email protected]
@@ -159,8 +170,11 @@ jobs:
           max_attempts: 2
           command:  bundle exec rake test:multiverse[group="${{ matrix.multiverse }}",verbose]
         env:
-          DB_PORT: 3306
+          DB_PASSWORD: root
           MYSQL_PASSWORD: root
+          DB_PORT: ${{ job.services.mysql.ports[3306] }}
+          MYSQL_PORT: ${{ job.services.mysql.ports[3306] }}
+          MYSQL_HOST: 127.0.0.1
           POSTGRES_USERNAME: postgres
           POSTGRES_PASSWORD: password
 
@@ -170,7 +184,7 @@ jobs:
 
   infinite_tracing:
     needs: build-ruby
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
       matrix:
@@ -198,6 +212,13 @@ jobs:
     needs: build-ruby
     runs-on: ubuntu-18.04
     services:
+      mysql:
+        image: mysql:5.7
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: yes
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
+        ports:
+          - "3306:3306"
       postgres:
         image: postgres:latest
         ports:
@@ -247,19 +268,16 @@ jobs:
         with:
           ruby-version: jruby-9.2.19.0
 
-      - name: Start mysql
-        run: sudo systemctl start mysql
-
       - name: Run Multiverse Tests
         uses: nick-invision/[email protected]
         with:
           timeout_minutes: 60
           max_attempts: 2
           command:  bundle exec rake test:multiverse[group=${{ matrix.multiverse }},verbose]
         env:
-          DB_PORT: 3306
-          MYSQL_PASSWORD: root
+          DB_PORT: ${{ job.services.mysql.ports[3306] }}
 
       - name: Annotate errors
         if: ${{ failure() }}
         uses: ./.github/actions/annotate
+
diff --git a/test/script/install_mysql55 b/test/script/install_mysql55
@@ -0,0 +1,17 @@
+#!/bin/bash
+set -u
+
+MYSQL55_PATH='/usr/local/mysql55'
+
+wget https://github.com/mysql/mysql-server/archive/refs/tags/mysql-5.5.63.tar.gz \
+&& tar xzf mysql-5.5.63.tar.gz \
+&& cd mysql-server-mysql-5.5.63/ \
+&& cmake . -DCMAKE_INSTALL_PREFIX=$MYSQL55_PATH \
+-DMYSQL_DATADIR=$MYSQL55_PATH/data \
+-DDOWNLOAD_BOOST=1 \
+-DWITH_BOOST=/tmp/boost \
+-DWITH_SSL=bundled \
+&& make \
+&& make install \
+&& cd .. \
+&& rm -rf mysql-5.5.63.tar.gz mysql-server-mysql-5.5.63