From 63a5106d458486a1373b43faaf8f30213715007f Mon Sep 17 00:00:00 2001
From: George Bosilca <gbosilca@nvidia.com>
Date: Tue, 3 Dec 2024 14:57:43 -0500
Subject: [PATCH] MPI_Wtime relative to MPI_Init

Issue #12953 identified an issue with MPI_Wtime taking the first call
as the source of relative timing. This patch moves the timing root
early in the MPI initialization (sessions or world model).

Fixes #12953.

Signed-off-by: George Bosilca <gbosilca@nvidia.com>
---
 ompi/instance/instance.c | 20 +++++++++++++++++++-
 ompi/mpi/c/wtime.c       | 10 +++-------
 2 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/ompi/instance/instance.c b/ompi/instance/instance.c
index 391c9bb03d1..9fd2d8561a0 100644
--- a/ompi/instance/instance.c
+++ b/ompi/instance/instance.c
@@ -7,6 +7,7 @@
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2023      Jeffrey M. Squyres.  All rights reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -53,10 +54,11 @@
 #include "ompi/mca/topo/base/base.h"
 #include "opal/mca/pmix/base/base.h"
 
-#include "opal/mca/mpool/base/mpool_base_tree.h"
 #include "ompi/mca/pml/base/pml_base_bsend.h"
 #include "ompi/util/timings.h"
+#include "opal/mca/mpool/base/mpool_base_tree.h"
 #include "opal/mca/pmix/pmix-internal.h"
+#include "opal/util/clock_gettime.h"
 
 ompi_predefined_instance_t ompi_mpi_instance_null = {{{{0}}}};
 
@@ -74,6 +76,14 @@ __opal_attribute_constructor__ static void instance_lock_init(void) {
 /** MPI_Init instance */
 ompi_instance_t *ompi_mpi_instance_default = NULL;
 
+/**
+ * @brief: Base timer initialization. All timers returned to the user via MPI_Wtime
+ *         are relative to this timer. Setting it early in during the common
+ *         initialization (world or session model) allows for measuring the cost of
+ *         the MPI initialization.
+ */
+struct timespec ompi_wtime_time_origin = {.tv_sec = 0};
+
 enum {
     OMPI_INSTANCE_INITIALIZING = -1,
     OMPI_INSTANCE_FINALIZING   = -2,
@@ -358,6 +368,14 @@ static int ompi_mpi_instance_init_common (int argc, char **argv)
     opal_pmix_lock_t mylock;
     OMPI_TIMING_INIT(64);
 
+    // We intentionally don't use the OPAL timer framework here.  See
+    // https://github.com/open-mpi/ompi/issues/3003 for more details.
+    struct timespec tp;
+    (void) opal_clock_gettime(&tp);
+    if (OPAL_UNLIKELY(0 == ompi_wtime_time_origin.tv_sec)) {
+        ompi_wtime_time_origin = tp;
+    }
+
     ret = ompi_mpi_instance_retain ();
     if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
         return ret;
diff --git a/ompi/mpi/c/wtime.c b/ompi/mpi/c/wtime.c
index b7918ad5d0d..1d7c672de59 100644
--- a/ompi/mpi/c/wtime.c
+++ b/ompi/mpi/c/wtime.c
@@ -15,6 +15,7 @@
  * Copyright (c) 2017      IBM Corporation.  All rights reserved.
  * Copyright (c) 2017      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2024      NVIDIA Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -42,15 +43,13 @@
 #pragma weak MPI_Wtime = PMPI_Wtime
 #endif
 #define MPI_Wtime PMPI_Wtime
+#endif
 /**
- * Have a base time set on the first call to wtime, to improve the range
+ * Use this as a base time set early during MPI initialization to improve the range
  * and accuracy of the user visible timer.
  * More info: https://github.com/mpi-forum/mpi-issues/issues/77#issuecomment-369663119
  */
-struct timespec ompi_wtime_time_origin = {.tv_sec = 0};
-#else  /* OMPI_BUILD_MPI_PROFILING */
 extern struct timespec ompi_wtime_time_origin;
-#endif
 
 double MPI_Wtime(void)
 {
@@ -62,9 +61,6 @@ double MPI_Wtime(void)
     // https://github.com/open-mpi/ompi/issues/3003 for more details.
     struct timespec tp;
     (void) opal_clock_gettime(&tp);
-    if (OPAL_UNLIKELY(0 == ompi_wtime_time_origin.tv_sec)) {
-        ompi_wtime_time_origin = tp;
-    }
     wtime  = (double)(tp.tv_nsec - ompi_wtime_time_origin.tv_nsec)/1.0e+9;
     wtime += (tp.tv_sec - ompi_wtime_time_origin.tv_sec);