possibly fixed issue #1704

src/app/orionld/orionld.cpp

@@ -1063,6 +1063,8 @@ int main(int argC, char* argV[])
 
   paParse(paArgs, argC, (char**) argV, 1, false);
 
+  LM_T(LmtMongoPool, ("DB Pool Size: %d", dbPoolSize));
+
   //
   // Config file
   //
@@ -1293,6 +1295,7 @@ int main(int argC, char* argV[])
   orionldTenantInit();
   orionldState.tenantP = &tenant0;
 
+  // mongocInit calls mongocTenantsGet => it gets total number of tenants from there and can change dbPoolSize
   mongocInit(dbURI, dbHost, dbUser, dbPwd, dbAuthDb, rplSet, dbAuthMechanism, dbSSL, dbCertFile);
 
   //

src/lib/logMsg/traceLevels.h

@@ -161,6 +161,7 @@ typedef enum TraceLevels
   LmtLegacy  = 220,                    // Old code (mongoBackend, json parsers, etc)
   LmtLegacySubMatch,                   // Old code - update/subscription match for subs/notifs
   LmtLegacySubCacheRefresh,            // Old code - sub-cache-refresh
+  LmtMongoPool,
 
   //
   // DDS

src/lib/mongoBackend/MongoGlobal.cpp

@@ -228,19 +228,14 @@ void mongoInit
 )
 {
   double tmo = timeout / 1000.0;  // milliseconds to float value in seconds
+
   if (!mongoStart(dbHost, dbName.c_str(), rplSet, user, pwd, mtenant, tmo, writeConcern, dbPoolSize, mutexTimeStat))
-  {
     LM_X(1, ("Fatal Error (MongoDB error)"));
-  }
 
   if (user[0] != 0)
-  {
     LM_I(("Connected to mongo at %s:%s as user '%s'", dbHost, dbName.c_str(), user));
-  }
   else
-  {
     LM_I(("Connected to mongo at %s:%s", dbHost, dbName.c_str()));
-  }
 
   setDbPrefix(dbName);
 
@@ -263,11 +258,13 @@ void mongoInit
     std::vector<std::string> orionDbs;
 
     getOrionDatabases(&orionDbs);
+    LM_T(LmtMongoPool, ("There are %d Orion databases, the connection pool size is: %d", orionDbs.size(), dbPoolSize));
 
     for (unsigned int ix = 0; ix < orionDbs.size(); ++ix)
     {
       std::string     orionDb     = orionDbs[ix];
       std::string     tenantName  = orionDb.substr(dbName.length() + 1);   // + 1 for the "_" in "orion_tenantA"
+
       OrionldTenant*  tenantP     = orionldTenantGet(tenantName.c_str());
 
       if (idIndex == true)
@@ -288,6 +285,7 @@ void mongoInit
 static void shutdownClient(void)
 {
   mongo::Status status = mongo::client::shutdown();
+
   if (!status.isOK())
   {
     LM_E(("Database Shutdown Error %s (cannot shutdown mongo client)", status.toString().c_str()));
@@ -320,20 +318,16 @@ bool mongoStart
   static bool alreadyDone = false;
 
   if (alreadyDone == true)
-  {
-    LM_E(("Runtime Error (mongoStart already called - can only be called once)"));
-    return false;
-  }
+    LM_RE(false, ("Runtime Error (mongoStart already called - can only be called once)"));
+
   alreadyDone = true;
 
   multitenant = _multitenant;
 
   mongo::Status status = mongo::client::initialize();
   if (!status.isOK())
-  {
-    LM_E(("Database Startup Error %s (cannot initialize mongo client)", status.toString().c_str()));
-    return false;
-  }
+    LM_RE(false, ("Database Startup Error %s (cannot initialize mongo client)", status.toString().c_str()));
+
   atexit(shutdownClient);
 
   if (mongoConnectionPoolInit(host,
@@ -347,7 +341,7 @@ bool mongoStart
                               poolSize,
                               semTimeStat) != 0)
   {
-    LM_E(("Database Startup Error (cannot initialize mongo connection pool)"));
+    LM_E(("Database Startup Error (unable to initialize the mongo connection pool)"));
     return false;
   }
 
@@ -356,14 +350,9 @@ bool mongoStart
 
 
 
-
-
 #ifdef UNIT_TEST
 
 static DBClientBase* connection = NULL;
-
-
-
 /* ****************************************************************************
 *
 * setMongoConnectionForUnitTest -

src/lib/mongoBackend/mongoConnectionPool.cpp

@@ -146,15 +146,9 @@ static DBClientBase* mongoConnect
       }
 
       if (tryNo == 0)
-      {
-        LM_E(("Database Startup Error (cannot connect to mongo - doing %d retries with a %d millisecond interval)",
-              retries,
-              RECONNECT_DELAY));
-      }
+        LM_E(("Database Startup Error (cannot connect to mongo - doing %d retries with a %d millisecond interval)", retries, RECONNECT_DELAY));
       else
-      {
         LM_T(LmtLegacy, ("Try %d connecting to mongo failed", tryNo));
-      }
 
       usleep(RECONNECT_DELAY * 1000);  // usleep accepts microseconds, RECONNECT_DELAY is in millis
     }
@@ -299,6 +293,8 @@ int mongoConnectionPoolInit
   //
   // Create the pool
   //
+  LM_T(LmtMongoPool, ("Creating a mongo connection pool of %d slots", poolSize));
+
   connectionPool  = (MongoConnection*) calloc(sizeof(MongoConnection), poolSize);
   if (connectionPool == NULL)
   {
@@ -383,9 +379,7 @@ DBClientBase* mongoPoolConnectionGet(void)
   struct timespec  diffTime;
 
   if (semStatistics)
-  {
     clock_gettime(CLOCK_REALTIME, &startTime);
-  }
 
   sem_wait(&connectionSem);
   sem_wait(&connectionPoolSem);
@@ -404,12 +398,16 @@ DBClientBase* mongoPoolConnectionGet(void)
     {
       connectionPool[ix].free = false;
       connection = connectionPool[ix].connection;
+      // LM_T(LmtMongoPool, ("Found a mongo connection in slot %d (%p)", ix, connection));
       break;
     }
   }
 
   sem_post(&connectionPoolSem);
 
+  if (connection == NULL)
+    LM_X(1, ("No mongo pool connection available"));
+
   return connection;
 }
 
@@ -421,19 +419,26 @@ DBClientBase* mongoPoolConnectionGet(void)
 */
 void mongoPoolConnectionRelease(DBClientBase* connection)
 {
+  bool ok = false;
+
   sem_wait(&connectionPoolSem);
 
   for (int ix = 0; ix < connectionPoolSize; ++ix)
   {
     if (connectionPool[ix].connection == connection)
     {
+      ok = true;
       connectionPool[ix].free = true;
       sem_post(&connectionSem);
+      // LM_T(LmtMongoPool, ("Releasing a mongo connection in slot %d (%p)", ix, connection));
       break;
     }
   }
 
   sem_post(&connectionPoolSem);
+
+  if (ok == false)
+    LM_T(LmtMongoPool, ("Not able to release mongo connection at %p", connection));
 }
 
 

src/lib/orionld/mongoc/mongocInit.cpp

@@ -326,9 +326,15 @@ void mongocInit
   //
   sem_init(&mongocConnectionSem, 0, 1);  // 0: shared between threads of the same process. 1: free to be taken
 
-  if (mongocTenantsGet() == false)
+  int tenants = 1;
+  if (mongocTenantsGet(&tenants) == false)
     LM_X(1, ("Unable to extract tenants from the database - fatal error"));
 
+  LM_T(LmtMongoPool, ("No of tenants: %d", tenants));
+  extern int dbPoolSize;
+  if (dbPoolSize < tenants)
+    dbPoolSize = tenants + 5;
+
   if (mongocGeoIndexInit() == false)
     LM_X(1, ("Unable to initialize geo indices in database - fatal error"));
 

src/lib/orionld/mongoc/mongocRegistrationsIter.cpp

@@ -36,6 +36,7 @@ extern "C"
 #include "orionld/common/orionldState.h"                         // orionldState
 #include "orionld/common/orionldError.h"                         // orionldError
 #include "orionld/mongoc/mongocWriteLog.h"                       // MONGOC_RLOG
+#include "orionld/mongoc/mongocConnectionRelease.h"              // mongocConnectionRelease
 #include "orionld/mongoc/mongocConnectionGet.h"                  // mongocConnectionGet
 #include "orionld/mongoc/mongocKjTreeFromBson.h"                 // mongocKjTreeFromBson
 #include "orionld/mongoc/mongocRegistrationsIter.h"              // RegCacheIterFunc
@@ -80,6 +81,7 @@ int mongocRegistrationsIter(RegCache* rcP, RegCacheIterFunc callback)
   if (mongoCursorP == NULL)
   {
     orionldError(OrionldInternalError, "Database Error", "mongoc_collection_find_with_opts ERROR", 500);
+    mongocConnectionRelease();
     mongoc_read_prefs_destroy(readPrefs);
     bson_destroy(&mongoFilter);
     return 1;
@@ -120,6 +122,7 @@ int mongocRegistrationsIter(RegCache* rcP, RegCacheIterFunc callback)
   mongoc_cursor_destroy(mongoCursorP);
   mongoc_read_prefs_destroy(readPrefs);
   bson_destroy(&mongoFilter);
+  mongocConnectionRelease();
 
   return retVal;
 }

src/lib/orionld/mongoc/mongocTenantsGet.cpp

@@ -45,7 +45,7 @@ extern "C"
 //
 // mongocTenantsGet -
 //
-bool mongocTenantsGet(void)
+bool mongocTenantsGet(int* noOfTenantsP)
 {
   bson_t        command;
   bson_t        reply;
@@ -94,6 +94,7 @@ bool mongocTenantsGet(void)
 
         if (orionldTenantLookup(tenantName) == NULL)
           orionldTenantCreate(tenantName, false, false);
+        *noOfTenantsP += 1;
       }
     }
   }

src/lib/orionld/mongoc/mongocTenantsGet.h

@@ -32,6 +32,6 @@
 //
 // mongocTenantsGet -
 //
-extern bool mongocTenantsGet(void);
+extern bool mongocTenantsGet(int* noOfTenantsP);
 
 #endif  // SRC_LIB_ORIONLD_MONGOC_MONGOCTENANTSGET_H_