From b6b9f96201161311e86f1b096746107b16dad88c Mon Sep 17 00:00:00 2001 From: Pascal Elahi Date: Wed, 16 Mar 2022 12:44:03 +0800 Subject: [PATCH 1/3] Added report of core binding VR reports where MPI ranks and OpenMP threads are bound. Initially tested on OSX, needs testing on Linux. --- src/allvars.h | 3 ++ src/main.cxx | 2 + src/proto.h | 15 ++++++ src/utilities.cxx | 113 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 133 insertions(+) diff --git a/src/allvars.h b/src/allvars.h index cb5ecf14..0e539c08 100644 --- a/src/allvars.h +++ b/src/allvars.h @@ -37,6 +37,9 @@ #include #include #include +#include +#include +#include #include #include diff --git a/src/main.cxx b/src/main.cxx index 236b8881..4311582a 100644 --- a/src/main.cxx +++ b/src/main.cxx @@ -83,6 +83,8 @@ void show_version_info(int argc, char *argv[]) #else << "no"; #endif + // report binding + report_binding(); } int run(int argc,char **argv) diff --git a/src/proto.h b/src/proto.h index e0e52258..7f8c4838 100644 --- a/src/proto.h +++ b/src/proto.h @@ -890,6 +890,21 @@ namespace vr { std::string GetMemUsage(const std::string &function); #define MEMORY_USAGE_REPORT(lvl) { if(LOG_ENABLED(lvl)) LOG(lvl) << GetMemUsage(__FUNCTION__); } +/// Core binding +//@{ +#ifdef __APPLE__ + +#define SYSCTL_CORE_COUNT "machdep.cpu.core_count" +#define CPU_SETSIZE 1024 +typedef struct cpu_set { + uint32_t count; +} cpu_set_t; +int sched_getaffinity(pid_t pid, size_t cpu_size, cpu_set_t *cpu_set); +#endif +void cpuset_to_cstr(cpu_set_t *mask, char *str); +void report_binding(); +//@} + namespace vr { /// Get the basename of `filename` std::string basename(const std::string &filename); diff --git a/src/utilities.cxx b/src/utilities.cxx index dbf1664a..9f1fa76e 100644 --- a/src/utilities.cxx +++ b/src/utilities.cxx @@ -86,6 +86,119 @@ std::string GetMemUsage(const std::string &function) return memory_report.str(); } +/* Borrowed from util-linux-2.13-pre7/schedutils/taskset.c */ +#ifdef __APPLE__ + +static inline void +CPU_ZERO(cpu_set_t *cs) { cs->count = 0; } + +static inline void +CPU_SET(int num, cpu_set_t *cs) { cs->count |= (1 << num); } + +static inline int +CPU_ISSET(int num, cpu_set_t *cs) { return (cs->count & (1 << num)); } + +int sched_getaffinity(pid_t pid, size_t cpu_size, cpu_set_t *cpu_set) +{ + int32_t core_count = 0; + size_t len = sizeof(core_count); + int ret = sysctlbyname(SYSCTL_CORE_COUNT, &core_count, &len, 0, 0); + if (ret) { + printf("error while get core count %d\n", ret); + return -1; + } + cpu_set->count = 0; + for (int i = 0; i < core_count; i++) { + cpu_set->count |= (1 << i); + } + + return 0; +} +#endif + +void cpuset_to_cstr(cpu_set_t *mask, char *str) +{ + char *ptr = str; + int i, j, entry_made = 0; + for (i = 0; i < CPU_SETSIZE; i++) { + if (CPU_ISSET(i, mask)) { + int run = 0; + entry_made = 1; + for (j = i + 1; j < CPU_SETSIZE; j++) { + if (CPU_ISSET(j, mask)) run++; + else break; + } + if (!run) + sprintf(ptr, "%d ", i); + else if (run == 1) { + sprintf(ptr, "%d,%d ", i, i + 1); + i++; + } else { + sprintf(ptr, "%d-%d ", i, i + run); + i += run; + } + while (*ptr != 0) ptr++; + } + } + ptr -= entry_made; + ptr = nullptr; +} + +void report_binding() +{ + // if there is no MPI do not report any binding +#if !defined(USEMPI) && !defined(USEOPENMP) + return; +#endif + int rank, size, thread; + string pinning_report = ""; + cpu_set_t coremask; + char clbuf[7 * CPU_SETSIZE], hnbuf[64]; +#ifdef USEMPI + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); + LOG_RANK0(info) <<"MPI Comm size = "< Date: Wed, 16 Mar 2022 13:27:30 +0800 Subject: [PATCH 2/3] Update to formating of core binding --- src/utilities.cxx | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/src/utilities.cxx b/src/utilities.cxx index 9f1fa76e..7a5f165c 100644 --- a/src/utilities.cxx +++ b/src/utilities.cxx @@ -150,38 +150,34 @@ void report_binding() #if !defined(USEMPI) && !defined(USEOPENMP) return; #endif - int rank, size, thread; - string pinning_report = ""; - cpu_set_t coremask; - char clbuf[7 * CPU_SETSIZE], hnbuf[64]; #ifdef USEMPI - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &size); - LOG_RANK0(info) <<"MPI Comm size = "< Date: Thu, 17 Mar 2022 15:11:28 +0800 Subject: [PATCH 3/3] Minor change, remove unnecessary for loop --- src/utilities.cxx | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/src/utilities.cxx b/src/utilities.cxx index 7a5f165c..e0b55040 100644 --- a/src/utilities.cxx +++ b/src/utilities.cxx @@ -161,34 +161,28 @@ void report_binding() memset(clbuf, 0, sizeof(clbuf)); memset(hnbuf, 0, sizeof(hnbuf)); (void)gethostname(hnbuf, sizeof(hnbuf)); - for (auto itask=0;itask