feat: retry initial MongoDB connections

package.json

@@ -48,6 +48,7 @@
     "mongodb": "~3.3.3",
     "node-fetch": "~2.6.0",
     "object-hash": "~1.3.1",
+    "promise-retry": "^1.1.1",
     "querystring": "~0.2.0",
     "ramda": "~0.26.1",
     "semver": "~6.3.0",

src/core/ReactionAPI.js

@@ -11,6 +11,7 @@ import Logger from "@reactioncommerce/logger";
 import appEvents from "./util/appEvents.js";
 import getAbsoluteUrl from "./util/getAbsoluteUrl.js";
 import initReplicaSet from "./util/initReplicaSet.js";
+import mongoConnectWithRetry from "./util/mongoConnectWithRetry.js";
 import config from "./config.js";
 import createApolloServer from "./createApolloServer.js";
 import coreResolvers from "./graphql/resolvers/index.js";
@@ -31,8 +32,6 @@ const {
 
 const debugLevels = ["DEBUG", "TRACE"];
 
-const { MongoClient } = mongodb;
-
 const optionsSchema = new SimpleSchema({
   "httpServer": {
     type: Object,
@@ -214,12 +213,7 @@ export default class ReactionAPI {
     const dbUrl = mongoUrl.slice(0, lastSlash);
     const dbName = mongoUrl.slice(lastSlash + 1);
 
-    const client = await MongoClient.connect(dbUrl, {
-      useNewUrlParser: true
-      // Uncomment this after this `mongodb` pkg bug is fixed:
-      // https://jira.mongodb.org/browse/NODE-2249
-      // useUnifiedTopology: true
-    });
+    const client = await mongoConnectWithRetry(dbUrl);
 
     this.mongoClient = client;
     this.setMongoDatabase(client.db(dbName));

src/core/util/initReplicaSet.js

@@ -1,8 +1,6 @@
 import { URL } from "url";
-import mongodb from "mongodb";
 import Logger from "@reactioncommerce/logger";
-
-const { MongoClient } = mongodb;
+import mongoConnectWithRetry from "./mongoConnectWithRetry.js";
 
 /**
  * @summary Sleep for some milliseconds
@@ -30,12 +28,7 @@ async function connect(parsedUrl) {
   dbParsedUrl.pathname = "";
   const dbUrl = dbParsedUrl.toString();
 
-  const client = await MongoClient.connect(dbUrl, {
-    useNewUrlParser: true
-    // Uncomment this after this `mongodb` pkg bug is fixed:
-    // https://jira.mongodb.org/browse/NODE-2249
-    // useUnifiedTopology: true
-  });
+  const client = await mongoConnectWithRetry(dbUrl);
 
   return {
     client,
@@ -51,31 +44,14 @@ async function connect(parsedUrl) {
  * @returns {Promise} indication of success/failure
  */
 export default async function initReplicaSet(mongoUrl) {
+  Logger.info("Initializing MongoDB replica set...");
   const parsedUrl = new URL(mongoUrl);
 
   // eventually we should set `stopped = true` when the developer interrupts
   // the process
   const stopped = false;
 
-  const canConnectTimestamp = Date.now();
-  let db;
-  let client;
-  while (!stopped) {
-    try {
-      // eslint-disable-next-line no-await-in-loop
-      ({ client, db } = await connect(parsedUrl));
-    } catch (error) {
-      if (Date.now() - canConnectTimestamp > 60000) {
-        Logger.error(error);
-        throw new Error("Unable to connect to MongoDB server after 1 minute");
-      }
-    }
-
-    if (client) break;
-
-    // eslint-disable-next-line no-await-in-loop
-    await sleep(100);
-  }
+  const { client, db } = await connect(parsedUrl);
 
   const replSetConfiguration = {
     _id: "rs0",
@@ -132,4 +108,5 @@ export default async function initReplicaSet(mongoUrl) {
   }
 
   await client.close();
+  Logger.info("Finished MongoDB replica set initialization");
 }

src/core/util/mongoConnectWithRetry.js

@@ -0,0 +1,45 @@
+import Logger from "@reactioncommerce/logger";
+import mongodb from "mongodb";
+import promiseRetry from "promise-retry";
+
+const { MongoClient } = mongodb;
+
+const mongoInitialConnectRetries = 10;
+
+/**
+ * @summary The MongoDB driver will auto-reconnect but not on the first connect.
+ *   If the first connection fails, it throws. Because we expect to be in a dynamic
+ *   Docker environment in which containers may start and slightly different times,
+ *   we want to try to reconnect for a bit, even on the first connect.
+ * @param {String} url MongoDB URL
+ * @return {Object} Client
+ */
+export default function mongoConnectWithRetry(url) {
+  return promiseRetry((retry, number) => {
+    if (number > 1) {
+      Logger.info(`Retrying connect to MongoDB... (${number - 1} of ${mongoInitialConnectRetries})`);
+    } else {
+      Logger.info("Connecting to MongoDB...");
+    }
+
+    return MongoClient.connect(url, {
+      useNewUrlParser: true
+      // Uncomment this after this `mongodb` pkg bug is fixed:
+      // https://jira.mongodb.org/browse/NODE-2249
+      // useUnifiedTopology: true
+    }).then((client) => {
+      Logger.info("Connected to MongoDB");
+      return client;
+    }).catch((error) => {
+      if (error.name === "MongoNetworkError") {
+        retry(error);
+      } else {
+        throw error;
+      }
+    });
+  }, {
+    factor: 1,
+    minTimeout: 3000,
+    retries: mongoInitialConnectRetries
+  }).catch((error) => { throw error; });
+}

src/index.js

@@ -21,4 +21,7 @@ async function runApp() {
   Logger.info(`GraphQL subscriptions ready at ${app.graphQLServerSubscriptionUrl} (port ${app.serverPort || "unknown"})`);
 }
 
-runApp().catch(Logger.error.bind(Logger));
+runApp().catch((error) => {
+  Logger.error(error);
+  process.exit(1); // eslint-disable-line no-process-exit
+});