PerfTools/JeProf: Make a library that allows a call to jemalloc profile heap dump function.

FWCore/Framework/bin/BuildFile.xml

@@ -83,3 +83,17 @@
   <use name="FWCore/ParameterSet"/>
   <use name="FWCore/ParameterSetReader"/>
 </bin>
+
+<bin name="cmsRunJEProf" file="cmsRun.cpp">
+  <use name="tbb"/>
+  <use name="boost"/>
+  <use name="boost_program_options"/>
+  <use name="jemalloc-prof"/>
+  <use name="FWCore/Framework"/>
+  <use name="FWCore/MessageLogger"/>
+  <use name="FWCore/PluginManager"/>
+  <use name="FWCore/ServiceRegistry"/>
+  <use name="FWCore/Utilities"/>
+  <use name="FWCore/ParameterSet"/>
+  <use name="FWCore/ParameterSetReader"/>
+</bin>

PerfTools/JeProf/BuildFile.xml

@@ -0,0 +1,5 @@
+<use name="sockets"/>
+<use name="FWCore/MessageLogger"/>
+<export>
+  <lib name="1"/>
+</export>

PerfTools/JeProf/interface/jeprof.h

@@ -0,0 +1,4 @@
+
+namespace cms::jeprof {
+  void makeHeapDump(const char *);
+}

PerfTools/JeProf/plugins/BuildFile.xml

@@ -2,6 +2,7 @@
 <use name="FWCore/ParameterSet"/>
 <use name="FWCore/Framework"/>
 <use name="FWCore/ServiceRegistry"/>
+<use name="PerfTools/JeProf"/>
 <library file="*.cc" name="JeProfPlugin">
   <flags EDM_PLUGIN="1"/>
 </library>

PerfTools/JeProf/plugins/JeProfService.cc

@@ -12,35 +12,12 @@
 #include "FWCore/Framework/interface/FileBlock.h"
 #include "FWCore/Framework/interface/Event.h"
 #include "FWCore/Framework/interface/Run.h"
+#include "PerfTools/JeProf/interface/jeprof.h"
 #include <string>
 #include <dlfcn.h>
 #include <cstdio>
 #include <cstring>
 
-extern "C" {
-typedef int (*mallctl_t)(const char *name, void *oldp, size_t *oldlenp, void *newp, size_t newlen);
-}
-
-namespace {
-  bool initialize_prof();
-
-  mallctl_t mallctl = nullptr;
-  const bool have_jemalloc_and_prof = initialize_prof();
-
-  bool initialize_prof() {
-    // check if mallctl and friends are available, if we are using jemalloc
-    mallctl = (mallctl_t)::dlsym(RTLD_DEFAULT, "mallctl");
-    if (mallctl == nullptr)
-      return false;
-    // check if heap profiling available, if --enable-prof was specified at build time
-    bool enable_prof = false;
-    size_t bool_s = sizeof(bool);
-    mallctl("prof.active", &enable_prof, &bool_s, nullptr, 0);
-    return enable_prof;
-  }
-
-}  // namespace
-
 namespace edm {
   class GlobalContext;
   class StreamContext;
@@ -123,10 +100,6 @@ using namespace edm::service;
 
 JeProfService::JeProfService(ParameterSet const &ps, ActivityRegistry &iRegistry)
     : mineventrecord_(1), prescale_(1), nrecord_(0), nevent_(0), nrun_(0), nlumi_(0), nfileopened_(0), nfileclosed_(0) {
-  if (!have_jemalloc_and_prof) {
-    edm::LogWarning("JeProfModule") << "JeProfModule requested but application is not"
-                                    << " currently being profiled with jemalloc profiling\n";
-  }
   // Get the configuration
   prescale_ = ps.getUntrackedParameter<int>("reportEventInterval", prescale_);
   mineventrecord_ = ps.getUntrackedParameter<int>("reportFirstEvent", mineventrecord_);
@@ -266,9 +239,6 @@ void JeProfService::postCloseFile(std::string const &) {
 }
 
 void JeProfService::makeDump(const std::string &format, std::string_view moduleLabel) {
-  if (!have_jemalloc_and_prof || format.empty())
-    return;
-
   std::string final(format);
   final = replace(final, "%I", nrecord_);
   final = replaceU64(final, "%E", nevent_);
@@ -278,7 +248,7 @@ void JeProfService::makeDump(const std::string &format, std::string_view moduleL
   final = replace(final, "%C", nfileclosed_);
   final = replace(final, "%M", moduleLabel);
   const char *fileName = final.c_str();
-  mallctl("prof.dump", nullptr, nullptr, &fileName, sizeof(const char *));
+  cms::jeprof::makeHeapDump(fileName);
 }
 
 std::string JeProfService::replace(const std::string &s, const char *pat, int val) {

PerfTools/JeProf/src/jeprof.cc

@@ -0,0 +1,48 @@
+#include "PerfTools/JeProf/interface/jeprof.h"
+#include "FWCore/MessageLogger/interface/MessageLogger.h"
+#include <string>
+#include <dlfcn.h>
+#include <cstdio>
+#include <cstring>
+#include <mutex>
+
+extern "C" {
+typedef int (*mallctl_t)(const char *name, void *oldp, size_t *oldlenp, void *newp, size_t newlen);
+}
+
+namespace {
+  bool initialize_prof();
+  mallctl_t mallctl = nullptr;
+  const bool have_jemalloc_and_prof = initialize_prof();
+
+  bool initialize_prof() {
+    // check if mallctl and friends are available, if we are using jemalloc
+    mallctl = (mallctl_t)::dlsym(RTLD_DEFAULT, "mallctl");
+    if (mallctl == nullptr)
+      return false;
+    // check if heap profiling available, if --enable-prof was specified at build time
+    bool enable_prof = false;
+    size_t bool_s = sizeof(bool);
+    mallctl("prof.active", &enable_prof, &bool_s, nullptr, 0);
+    return enable_prof;
+  }
+}  // namespace
+
+namespace cms::jeprof {
+  std::once_flag warning_flag;
+  void makeHeapDump(const char *fileName) {
+    std::once_flag warning_flag;
+    if (!have_jemalloc_and_prof) {
+      std::call_once(warning_flag,
+                     []() {
+                       edm::LogWarning("JeProfModule")
+                           << "JeProfModule requested but application is not"
+                           << " currently being profiled with jemalloc profiling enabled\n"
+                           << "Enable jemalloc profiling by running\n"
+                           << "MALLOC_CONF=prof_leak:true,lg_prof_sample:10,prof_final:true cmsRunJEProf config.py\n";
+                     });
+      return;
+    }
+    mallctl("prof.dump", nullptr, nullptr, &fileName, sizeof(const char *));
+  }
+}  // namespace cms::jeprof

PerfTools/JeProf/test/BuildFile.xml

@@ -0,0 +1,8 @@
+<bin name="jeprof_warn_t" file="jeprof_t.cc">
+  <use name="jemalloc"/>
+  <use name="PerfTools/JeProf"/>
+</bin>
+<bin name="jeprof_nowarn_t" file="jeprof_t.cc">
+  <use name="jemalloc-prof"/>
+  <use name="PerfTools/JeProf"/>
+</bin>

PerfTools/JeProf/test/jeprof_t.cc

@@ -0,0 +1,9 @@
+#include "PerfTools/JeProf/interface/jeprof.h"
+#include <string>
+
+int main() {
+  setenv("MALLOC_CONF", "prof_leak:true,lg_prof_sample:10,prof_final:true", 1);
+  std::string name("heap.dump");
+  const char *fileName = name.c_str();
+  cms::jeprof::makeHeapDump(fileName);
+}