fix(NODE-3528): add support for snappy v7 (#2947)

mongodb · Aug 25, 2021 · 54f5c2d · 54f5c2d
1 parent cc0cc20
commit 54f5c2d
Show file tree

Hide file tree

Showing 9 changed files with 6,739 additions and 92 deletions.
diff --git a/.evergreen/config.yml b/.evergreen/config.yml
@@ -480,6 +480,17 @@ functions:
           rm -f ./prepare_client_encryption.sh
 
           UNIFIED=${UNIFIED} MONGODB_URI="${MONGODB_URI}" bash ${PROJECT_DIRECTORY}/.evergreen/run-custom-csfle-tests.sh
+  run custom snappy tests:
+    - command: subprocess.exec
+      params:
+        working_dir: src
+        timeout_secs: 60
+        env:
+          MONGODB_URI: '${MONGODB_URI}'
+          PROJECT_DIRECTORY: '${PROJECT_DIRECTORY}'
+        binary: bash
+        args:
+          - '${PROJECT_DIRECTORY}/.evergreen/run-snappy-version-test.sh'
   upload test results:
     - command: attach.xunit_results
       params:
@@ -1545,16 +1556,34 @@ tasks:
       - func: run checks
   - name: run-custom-csfle-tests
     tags:
-      - run-custom-csfle-tests
+      - run-custom-dependency-tests
     commands:
       - func: install dependencies
         vars:
           NODE_LTS_NAME: erbium
       - func: bootstrap mongo-orchestration
         vars:
-          VERSION: '4.4'
+          VERSION: '5.0'
           TOPOLOGY: server
       - func: run custom csfle tests
+        vars:
+          UNIFIED: 1
+          NODE_LTS_NAME: erbium
+  - name: run-custom-snappy-tests
+    tags:
+      - run-custom-dependency-tests
+    commands:
+      - func: install dependencies
+        vars:
+          NODE_LTS_NAME: erbium
+      - func: bootstrap mongo-orchestration
+        vars:
+          VERSION: '5.0'
+          TOPOLOGY: server
+      - func: run custom snappy tests
+        vars:
+          UNIFIED: 1
+          NODE_LTS_NAME: erbium
 buildvariants:
   - name: macos-1014-fermium
     display_name: macOS 10.14 Node Fermium
@@ -2109,11 +2138,12 @@ buildvariants:
     run_on: rhel70
     tasks:
       - run-checks
-  - name: ubuntu1804-custom-csfle-tests
-    display_name: Custom FLE Version Test
-    run_on: ubuntu1804-test
+  - name: ubuntu1804-custom-dependency-tests
+    display_name: Custom Dependency Version Test
+    run_on: ubuntu1804-large
     tasks:
       - run-custom-csfle-tests
+      - run-custom-snappy-tests
   - name: ubuntu1804-test-mongodb-aws
     display_name: MONGODB-AWS Auth test
     run_on: ubuntu1804-test

diff --git a/.evergreen/config.yml.in b/.evergreen/config.yml.in
@@ -522,6 +522,18 @@ functions:
 
           UNIFIED=${UNIFIED} MONGODB_URI="${MONGODB_URI}" bash ${PROJECT_DIRECTORY}/.evergreen/run-custom-csfle-tests.sh
 
+  "run custom snappy tests":
+    - command: subprocess.exec
+      params:
+        working_dir: "src"
+        timeout_secs: 60
+        env:
+          MONGODB_URI: ${MONGODB_URI}
+          PROJECT_DIRECTORY: ${PROJECT_DIRECTORY}
+        binary: bash
+        args:
+          - "${PROJECT_DIRECTORY}/.evergreen/run-snappy-version-test.sh"
+
   "upload test results":
     - command: attach.xunit_results
       params:

diff --git a/.evergreen/generate_evergreen_tasks.js b/.evergreen/generate_evergreen_tasks.js
@@ -74,10 +74,7 @@ const OPERATING_SYSTEMS = [
   )
 );
 
-const WINDOWS_SKIP_TAGS = new Set([
-  'atlas-connect',
-  'auth'
-]);
+const WINDOWS_SKIP_TAGS = new Set(['atlas-connect', 'auth']);
 
 const BASE_TASKS = [];
 const TASKS = [];
@@ -109,9 +106,7 @@ function makeTask({ mongoVersion, topology }) {
 }
 
 MONGODB_VERSIONS.forEach(mongoVersion => {
-  TOPOLOGIES.forEach(topology =>
-    BASE_TASKS.push(makeTask({ mongoVersion, topology }))
-  );
+  TOPOLOGIES.forEach(topology => BASE_TASKS.push(makeTask({ mongoVersion, topology })));
 });
 
 TASKS.push(
@@ -125,7 +120,8 @@ TASKS.push(
     tags: ['auth', 'kerberos', 'legacy'],
     commands: [
       { func: 'install dependencies' },
-      { func: 'run kerberos tests',
+      {
+        func: 'run kerberos tests',
         vars: {
           UNIFIED: 0
         }
@@ -137,7 +133,8 @@ TASKS.push(
     tags: ['auth', 'kerberos', 'unified'],
     commands: [
       { func: 'install dependencies' },
-      { func: 'run kerberos tests',
+      {
+        func: 'run kerberos tests',
         vars: {
           UNIFIED: 1
         }
@@ -296,7 +293,7 @@ OCSP_VERSIONS.forEach(VERSION => {
 const AWS_AUTH_TASKS = [];
 
 AWS_AUTH_VERSIONS.forEach(VERSION => {
-  const name = (ex) => `aws-${VERSION}-auth-test-${ex.split(' ').join('-')}`;
+  const name = ex => `aws-${VERSION}-auth-test-${ex.split(' ').join('-')}`;
   // AWS_AUTH_TASKS.push(name);
 
   const aws_funcs = [
@@ -325,10 +322,10 @@ AWS_AUTH_VERSIONS.forEach(VERSION => {
       { func: 'setup aws env' },
       fn
     ]
-  }))
+  }));
 
   TASKS.push(...aws_tasks);
-  AWS_AUTH_TASKS.push(...aws_tasks.map(t => t.name))
+  AWS_AUTH_TASKS.push(...aws_tasks.map(t => t.name));
 });
 
 const BUILD_VARIANTS = [];
@@ -342,31 +339,33 @@ const getTaskList = (() => {
       return memo[key];
     }
     const taskList = BASE_TASKS.concat(TASKS);
-    const ret = taskList.filter(task => {
-      if (task.name.match(/^aws/)) return false;
+    const ret = taskList
+      .filter(task => {
+        if (task.name.match(/^aws/)) return false;
 
-      // skip unsupported tasks on windows
-      if (os.match(/^windows/) && task.tags.filter(tag => WINDOWS_SKIP_TAGS.has(tag)).length) {
-        return false;
-      }
+        // skip unsupported tasks on windows
+        if (os.match(/^windows/) && task.tags.filter(tag => WINDOWS_SKIP_TAGS.has(tag)).length) {
+          return false;
+        }
 
-      const tasksWithVars = task.commands.filter(task => !!task.vars);
-      if (!tasksWithVars.length) {
-        return true;
-      }
+        const tasksWithVars = task.commands.filter(task => !!task.vars);
+        if (!tasksWithVars.length) {
+          return true;
+        }
 
-      // kerberos tests don't require mongo orchestration
-      if (task.tags.filter(tag => tag === 'kerberos').length) {
-        return true;
-      }
+        // kerberos tests don't require mongo orchestration
+        if (task.tags.filter(tag => tag === 'kerberos').length) {
+          return true;
+        }
 
-      const { VERSION } = tasksWithVars[0].vars || {};
-      if (VERSION === 'latest') {
-        return semver.satisfies(semver.coerce(LATEST_EFFECTIVE_VERSION), mongoVersion);
-      }
+        const { VERSION } = tasksWithVars[0].vars || {};
+        if (VERSION === 'latest') {
+          return semver.satisfies(semver.coerce(LATEST_EFFECTIVE_VERSION), mongoVersion);
+        }
 
-      return semver.satisfies(semver.coerce(VERSION), mongoVersion);
-    }).map(x => x.name);
+        return semver.satisfies(semver.coerce(VERSION), mongoVersion);
+      })
+      .map(x => x.name);
 
     memo[key] = ret;
     return ret;
@@ -419,38 +418,48 @@ SINGLETON_TASKS.push({
   ]
 });
 
-SINGLETON_TASKS.push({
-  name: 'run-custom-csfle-tests',
-  tags: ['run-custom-csfle-tests'],
+const oneOffFuncs = [
+  { func: 'run custom csfle tests', vars: { UNIFIED: 1, NODE_LTS_NAME: 'erbium' } },
+  { func: 'run custom snappy tests', vars: { UNIFIED: 1, NODE_LTS_NAME: 'erbium' } }
+];
+
+const oneOffFuncAsTasks = oneOffFuncs.map(oneOffFunc => ({
+  name: `${oneOffFunc.func.split(' ').join('-')}`,
+  tags: ['run-custom-dependency-tests'],
   commands: [
     {
       func: 'install dependencies',
       vars: {
-        NODE_LTS_NAME: 'erbium',
-      },
+        NODE_LTS_NAME: 'erbium'
+      }
     },
     {
       func: 'bootstrap mongo-orchestration',
       vars: {
-        VERSION: '4.4',
+        VERSION: '5.0',
         TOPOLOGY: 'server'
       }
     },
-    { func: 'run custom csfle tests' }
+    oneOffFunc
   ]
-});
+}));
 
-BUILD_VARIANTS.push({
-  name: 'lint',
-  display_name: 'lint',
-  run_on: 'rhel70',
-  tasks: ['run-checks']
-}, {
-  name: 'ubuntu1804-custom-csfle-tests',
-  display_name: 'Custom FLE Version Test',
-  run_on: 'ubuntu1804-test',
-  tasks: ['run-custom-csfle-tests']
-});
+SINGLETON_TASKS.push(...oneOffFuncAsTasks);
+
+BUILD_VARIANTS.push(
+  {
+    name: 'lint',
+    display_name: 'lint',
+    run_on: 'rhel70',
+    tasks: ['run-checks']
+  },
+  {
+    name: 'ubuntu1804-custom-dependency-tests',
+    display_name: 'Custom Dependency Version Test',
+    run_on: 'ubuntu1804-large',
+    tasks: oneOffFuncAsTasks.map(({ name }) => name)
+  }
+);
 
 // special case for MONGODB-AWS authentication
 BUILD_VARIANTS.push({
@@ -464,7 +473,7 @@ BUILD_VARIANTS.push({
 });
 
 const fileData = yaml.safeLoad(fs.readFileSync(`${__dirname}/config.yml.in`, 'utf8'));
-fileData.tasks = (fileData.tasks || []).concat(BASE_TASKS).concat(TASKS).concat(SINGLETON_TASKS);
+fileData.tasks = (fileData.tasks || []).concat(BASE_TASKS, TASKS, SINGLETON_TASKS);
 fileData.buildvariants = (fileData.buildvariants || []).concat(BUILD_VARIANTS);
 
 fs.writeFileSync(`${__dirname}/config.yml`, yaml.safeDump(fileData, { lineWidth: 120 }), 'utf8');
diff --git a/.evergreen/init-nvm.sh b/.evergreen/init-nvm.sh
@@ -0,0 +1,19 @@
+#! /usr/bin/env bash
+
+export PATH="/opt/mongodbtoolchain/v2/bin:$PATH"
+NODE_ARTIFACTS_PATH="${PROJECT_DIRECTORY}/node-artifacts"
+export NVM_DIR="${NODE_ARTIFACTS_PATH}/nvm"
+
+if [[ "$OS" == "Windows_NT" ]]; then
+    NVM_HOME=$(cygpath -w "$NVM_DIR")
+    export NVM_HOME
+    NVM_SYMLINK=$(cygpath -w "$NODE_ARTIFACTS_PATH/bin")
+    export NVM_SYMLINK
+    NVM_ARTIFACTS_PATH=$(cygpath -w "$NODE_ARTIFACTS_PATH/bin")
+    export NVM_ARTIFACTS_PATH
+    PATH=$(cygpath $NVM_SYMLINK):$(cygpath $NVM_HOME):$PATH
+    export PATH
+    echo "updated path on windows PATH=$PATH"
+else
+    [ -s "$NVM_DIR/nvm.sh" ] && source "$NVM_DIR/nvm.sh"
+fi
diff --git a/.evergreen/run-snappy-version-test.sh b/.evergreen/run-snappy-version-test.sh
@@ -0,0 +1,12 @@
+#! /usr/bin/env bash
+
+source "${PROJECT_DIRECTORY}/.evergreen/init-nvm.sh"
+export MONGODB_URI="${MONGODB_URI}"
+
+npm i --no-save snappy@6
+
+npx mocha test/unit/snappy.test.js
+
+npm i --no-save snappy@7
+
+npx mocha test/unit/snappy.test.js
diff --git a/lib/core/connection/utils.js b/lib/core/connection/utils.js
@@ -1,5 +1,8 @@
 'use strict';
 
+const parsePackageVersion = require('../../utils').parsePackageVersion;
+const MongoError = require('../error').MongoError;
+
 const require_optional = require('optional-require')(require);
 
 function debugOptions(debugFields, options) {
@@ -16,7 +19,15 @@ function retrieveBSON() {
   BSON.native = false;
 
   const optionalBSON = require_optional('bson-ext');
+  const bsonExtVersion = parsePackageVersion(
+    require_optional('bson-ext/package.json') || { version: '0.0.0' }
+  );
   if (optionalBSON) {
+    if (bsonExtVersion.major >= 4) {
+      throw new MongoError(
+        'bson-ext version 4 and above does not work with the 3.x version of the mongodb driver'
+      );
+    }
     optionalBSON.native = true;
     return optionalBSON;
   }
@@ -31,21 +42,43 @@ function noSnappyWarning() {
   );
 }
 
+const PKG_VERSION = Symbol('kPkgVersion');
+
 // Facilitate loading Snappy optionally
 function retrieveSnappy() {
-  let snappy = require_optional('snappy');
+  const snappy = require_optional('snappy');
   if (!snappy) {
-    snappy = {
+    return {
       compress: noSnappyWarning,
       uncompress: noSnappyWarning,
       compressSync: noSnappyWarning,
       uncompressSync: noSnappyWarning
     };
   }
+
+  const snappyPkg = require_optional('snappy/package.json') || { version: '0.0.0' };
+  const version = parsePackageVersion(snappyPkg);
+  snappy[PKG_VERSION] = version;
+  if (version.major >= 7) {
+    const compressOriginal = snappy.compress;
+    const uncompressOriginal = snappy.uncompress;
+    snappy.compress = (data, callback) => {
+      compressOriginal(data)
+        .then(res => callback(undefined, res))
+        .catch(error => callback(error));
+    };
+    snappy.uncompress = (data, callback) => {
+      uncompressOriginal(data)
+        .then(res => callback(undefined, res))
+        .catch(error => callback(error));
+    };
+  }
+
   return snappy;
 }
 
 module.exports = {
+  PKG_VERSION,
   debugOptions,
   retrieveBSON,
   retrieveSnappy