From 91ab7ebe2617b3dc0dd89b63d48efd47dcae3965 Mon Sep 17 00:00:00 2001 From: anagainaru Date: Mon, 27 Nov 2023 19:22:04 -0500 Subject: [PATCH 01/18] Revert "Only rank 0 should print the initialization message in perfstub" This reverts commit 39f0cb2f43a8ba424dfc1ef78d95aa33e6d4b025. --- source/adios2/core/ADIOS.cpp | 2 +- thirdparty/perfstubs/adios2-perfstubs-interface.h.in | 2 +- thirdparty/perfstubs/perfstubs/perfstubs_api/timer.c | 9 ++++----- thirdparty/perfstubs/perfstubs/perfstubs_api/timer.h | 6 +++--- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/source/adios2/core/ADIOS.cpp b/source/adios2/core/ADIOS.cpp index d7227a8e40..3e058c9f20 100644 --- a/source/adios2/core/ADIOS.cpp +++ b/source/adios2/core/ADIOS.cpp @@ -115,7 +115,7 @@ ADIOS::ADIOS(const std::string configFile, helper::Comm comm, const std::string static bool perfstubsInit(false); if (!perfstubsInit) { - PERFSTUBS_INITIALIZE(m_Comm.Rank()); + PERFSTUBS_INITIALIZE(); perfstubsInit = true; atexit(ps_finalize_); } diff --git a/thirdparty/perfstubs/adios2-perfstubs-interface.h.in b/thirdparty/perfstubs/adios2-perfstubs-interface.h.in index d108329c5e..021c3118b0 100644 --- a/thirdparty/perfstubs/adios2-perfstubs-interface.h.in +++ b/thirdparty/perfstubs/adios2-perfstubs-interface.h.in @@ -6,7 +6,7 @@ #else -#define PERFSTUBS_INITIALIZE(rank) +#define PERFSTUBS_INITIALIZE() #define PERFSTUBS_FINALIZE() #define PERFSTUBS_PAUSE_MEASUREMENT() #define PERFSTUBS_RESUME_MEASUREMENT() diff --git a/thirdparty/perfstubs/perfstubs/perfstubs_api/timer.c b/thirdparty/perfstubs/perfstubs/perfstubs_api/timer.c index 8b62fa0c32..b4177e5521 100644 --- a/thirdparty/perfstubs/perfstubs/perfstubs_api/timer.c +++ b/thirdparty/perfstubs/perfstubs/perfstubs_api/timer.c @@ -102,7 +102,7 @@ PS_WEAK_PRE void ps_tool_free_counter_data(ps_tool_counter_data_t *) PS_WEAK_POS PS_WEAK_PRE void ps_tool_free_metadata(ps_tool_metadata_t *) PS_WEAK_POST; #endif -void initialize_library(const int rank) { +void initialize_library() { #ifdef PERFSTUBS_USE_STATIC /* The initialization function is the only required one */ initialize_functions[0] = &ps_tool_initialize; @@ -110,8 +110,7 @@ void initialize_library(const int rank) { perfstubs_initialized = PERFSTUBS_FAILURE; return; } - if (rank == 0) - printf("Found ps_tool_initialize(), registering tool\n"); + printf("Found ps_tool_initialize(), registering tool\n"); finalize_functions[0] = &ps_tool_finalize; pause_measurement_functions[0] = &ps_tool_pause_measurement; resume_measurement_functions[0] = &ps_tool_resume_measurement; @@ -219,13 +218,13 @@ static inline void ps_register_thread_internal(void) { } /* Initialization */ -void ps_initialize_(const int rank) { +void ps_initialize_(void) { int i; /* Only do this once */ if (perfstubs_initialized != PERFSTUBS_UNKNOWN) { return; } - initialize_library(rank); + initialize_library(); for (i = 0 ; i < num_tools_registered ; i++) { initialize_functions[i](); } diff --git a/thirdparty/perfstubs/perfstubs/perfstubs_api/timer.h b/thirdparty/perfstubs/perfstubs/perfstubs_api/timer.h index 1e5af3d2e2..c806c9ed64 100644 --- a/thirdparty/perfstubs/perfstubs/perfstubs_api/timer.h +++ b/thirdparty/perfstubs/perfstubs/perfstubs_api/timer.h @@ -49,7 +49,7 @@ extern int perfstubs_initialized; extern "C" { #endif -void ps_initialize_(const int rank); +void ps_initialize_(void); void ps_finalize_(void); void ps_pause_measurement_(void); void ps_resume_measurement_(void); @@ -93,7 +93,7 @@ char* ps_make_timer_name_(const char * file, const char * func, int line); * line or in a config.h file, however your project does it */ -#define PERFSTUBS_INITIALIZE(rank) ps_initialize_(rank); +#define PERFSTUBS_INITIALIZE() ps_initialize_(); #define PERFSTUBS_FINALIZE() ps_finalize_(); @@ -170,7 +170,7 @@ char* ps_make_timer_name_(const char * file, const char * func, int line); #else // defined(PERFSTUBS_USE_TIMERS) -#define PERFSTUBS_INITIALIZE(rank) +#define PERFSTUBS_INITIALIZE() #define PERFSTUBS_FINALIZE() #define PERFSTUBS_PAUSE_MEASUREMENT() #define PERFSTUBS_RESUME_MEASUREMENT() From 735bbfc8f44c0d99069d070587fc8a80c209fba1 Mon Sep 17 00:00:00 2001 From: Greg Eisenhauer Date: Tue, 28 Nov 2023 08:57:21 -0500 Subject: [PATCH 02/18] perfstubs 2023-11-27 (845d0702) (#3944) Code extracted from: https://github.com/khuck/perfstubs.git at commit 845d070265c7ab1f995edec18ecb86a5d954b046 (master). Upstream Shortlog ----------------- Co-authored-by: PerfStubs Upstream --- thirdparty/perfstubs/perfstubs/LICENSE | 2 +- thirdparty/perfstubs/perfstubs/README.md | 2 +- .../perfstubs/perfstubs_api/README.md | 6 +- .../perfstubs/perfstubs_api/config.h.in | 2 +- .../perfstubs/perfstubs/perfstubs_api/timer.c | 381 +++++++----------- .../perfstubs/perfstubs/perfstubs_api/timer.h | 17 +- .../perfstubs/perfstubs/perfstubs_api/tool.h | 2 +- 7 files changed, 171 insertions(+), 241 deletions(-) diff --git a/thirdparty/perfstubs/perfstubs/LICENSE b/thirdparty/perfstubs/perfstubs/LICENSE index b10754dbe6..667733e1f6 100644 --- a/thirdparty/perfstubs/perfstubs/LICENSE +++ b/thirdparty/perfstubs/perfstubs/LICENSE @@ -1,6 +1,6 @@ BSD 3-Clause License -Copyright (c) 2019-2020, Kevin Huck +Copyright (c) 2019-2022, Kevin Huck and University of Oregon All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/thirdparty/perfstubs/perfstubs/README.md b/thirdparty/perfstubs/perfstubs/README.md index 75f77ec1bb..60bcf5565f 100644 --- a/thirdparty/perfstubs/perfstubs/README.md +++ b/thirdparty/perfstubs/perfstubs/README.md @@ -1,6 +1,6 @@ # PerfStubs -Copyright (c) 2019-2020 University of Oregon +Copyright (c) 2019-2022 University of Oregon Distributed under the BSD Software License (See accompanying file LICENSE.txt) diff --git a/thirdparty/perfstubs/perfstubs/perfstubs_api/README.md b/thirdparty/perfstubs/perfstubs/perfstubs_api/README.md index b0a86dfb3e..cb1579599c 100644 --- a/thirdparty/perfstubs/perfstubs/perfstubs_api/README.md +++ b/thirdparty/perfstubs/perfstubs/perfstubs_api/README.md @@ -1,6 +1,6 @@ -# Profiling Interface for ADIOS2 +# Profiling Interface for Libraries and Applications -Copyright (c) 2019-2020 University of Oregon +Copyright (c) 2019-2022 University of Oregon Distributed under the BSD Software License (See accompanying file LICENSE.txt) @@ -10,7 +10,7 @@ This is a generic design and implementation for other libraries and tools. ## Todo Items - [x] Make the interface generic. - [x] Replace ADIOST-specific symbols with generic versions that will be - implemented by interested measurement libraries (i.e. Score-P). + implemented by interested measurement libraries (i.e. Score-P). - ~~[ ] New environment variable specifying location of library containing function implementations.~~ - [x] Remove dynamic-linking specific approach (checking ```LD_PRELOAD```) diff --git a/thirdparty/perfstubs/perfstubs/perfstubs_api/config.h.in b/thirdparty/perfstubs/perfstubs/perfstubs_api/config.h.in index 222b53cc62..afc84f1418 100644 --- a/thirdparty/perfstubs/perfstubs/perfstubs_api/config.h.in +++ b/thirdparty/perfstubs/perfstubs/perfstubs_api/config.h.in @@ -1,5 +1,5 @@ // the configured options and settings for Tutorial -// Copyright (c) 2019-2020 University of Oregon +// Copyright (c) 2019-2022 University of Oregon // Distributed under the BSD Software License // (See accompanying file LICENSE.txt) #define PerfStubs_VERSION_MAJOR @PerfStubs_VERSION_MAJOR@ diff --git a/thirdparty/perfstubs/perfstubs/perfstubs_api/timer.c b/thirdparty/perfstubs/perfstubs/perfstubs_api/timer.c index b4177e5521..d219ea5e5b 100644 --- a/thirdparty/perfstubs/perfstubs/perfstubs_api/timer.c +++ b/thirdparty/perfstubs/perfstubs/perfstubs_api/timer.c @@ -1,22 +1,20 @@ -// Copyright (c) 2019-2020 University of Oregon +// Copyright (c) 2019-2022 University of Oregon // Distributed under the BSD Software License // (See accompanying file LICENSE.txt) #ifndef _GNU_SOURCE #define _GNU_SOURCE // needed to define RTLD_DEFAULT #endif +#include #include #include #include -#include #include "pthread.h" #ifndef PERFSTUBS_USE_TIMERS #define PERFSTUBS_USE_TIMERS #endif #include "perfstubs_api/timer.h" -#define MAX_TOOLS 1 - /* Make sure that the Timer singleton is constructed when the * library is loaded. This will ensure (on linux, anyway) that * we can assert that we have m_Initialized on the main thread. */ @@ -39,30 +37,30 @@ static void make_key(void) { /* Function pointers */ -ps_initialize_t initialize_functions[MAX_TOOLS]; -ps_finalize_t finalize_functions[MAX_TOOLS]; -ps_pause_measurement_t pause_measurement_functions[MAX_TOOLS]; -ps_resume_measurement_t resume_measurement_functions[MAX_TOOLS]; -ps_register_thread_t register_thread_functions[MAX_TOOLS]; -ps_dump_data_t dump_data_functions[MAX_TOOLS]; -ps_timer_create_t timer_create_functions[MAX_TOOLS]; -ps_timer_start_t timer_start_functions[MAX_TOOLS]; -ps_timer_stop_t timer_stop_functions[MAX_TOOLS]; -ps_start_string_t start_string_functions[MAX_TOOLS]; -ps_stop_string_t stop_string_functions[MAX_TOOLS]; -ps_stop_current_t stop_current_functions[MAX_TOOLS]; -ps_set_parameter_t set_parameter_functions[MAX_TOOLS]; -ps_dynamic_phase_start_t dynamic_phase_start_functions[MAX_TOOLS]; -ps_dynamic_phase_stop_t dynamic_phase_stop_functions[MAX_TOOLS]; -ps_create_counter_t create_counter_functions[MAX_TOOLS]; -ps_sample_counter_t sample_counter_functions[MAX_TOOLS]; -ps_set_metadata_t set_metadata_functions[MAX_TOOLS]; -ps_get_timer_data_t get_timer_data_functions[MAX_TOOLS]; -ps_get_counter_data_t get_counter_data_functions[MAX_TOOLS]; -ps_get_metadata_t get_metadata_functions[MAX_TOOLS]; -ps_free_timer_data_t free_timer_data_functions[MAX_TOOLS]; -ps_free_counter_data_t free_counter_data_functions[MAX_TOOLS]; -ps_free_metadata_t free_metadata_functions[MAX_TOOLS]; +ps_initialize_t initialize_function; +ps_finalize_t finalize_function; +ps_pause_measurement_t pause_measurement_function; +ps_resume_measurement_t resume_measurement_function; +ps_register_thread_t register_thread_function; +ps_dump_data_t dump_data_function; +ps_timer_create_t timer_create_function; +ps_timer_start_t timer_start_function; +ps_timer_stop_t timer_stop_function; +ps_start_string_t start_string_function; +ps_stop_string_t stop_string_function; +ps_stop_current_t stop_current_function; +ps_set_parameter_t set_parameter_function; +ps_dynamic_phase_start_t dynamic_phase_start_function; +ps_dynamic_phase_stop_t dynamic_phase_stop_function; +ps_create_counter_t create_counter_function; +ps_sample_counter_t sample_counter_function; +ps_set_metadata_t set_metadata_function; +ps_get_timer_data_t get_timer_data_function; +ps_get_counter_data_t get_counter_data_function; +ps_get_metadata_t get_metadata_function; +ps_free_timer_data_t free_timer_data_function; +ps_free_counter_data_t free_counter_data_function; +ps_free_metadata_t free_metadata_function; #ifdef PERFSTUBS_USE_STATIC @@ -102,93 +100,94 @@ PS_WEAK_PRE void ps_tool_free_counter_data(ps_tool_counter_data_t *) PS_WEAK_POS PS_WEAK_PRE void ps_tool_free_metadata(ps_tool_metadata_t *) PS_WEAK_POST; #endif -void initialize_library() { +void initialize_library(void) { #ifdef PERFSTUBS_USE_STATIC /* The initialization function is the only required one */ - initialize_functions[0] = &ps_tool_initialize; - if (initialize_functions[0] == NULL) { + initialize_function = &ps_tool_initialize; + if (initialize_function == NULL) { perfstubs_initialized = PERFSTUBS_FAILURE; return; } - printf("Found ps_tool_initialize(), registering tool\n"); - finalize_functions[0] = &ps_tool_finalize; - pause_measurement_functions[0] = &ps_tool_pause_measurement; - resume_measurement_functions[0] = &ps_tool_resume_measurement; - register_thread_functions[0] = &ps_tool_register_thread; - dump_data_functions[0] = &ps_tool_dump_data; - timer_create_functions[0] = &ps_tool_timer_create; - timer_start_functions[0] = &ps_tool_timer_start; - timer_stop_functions[0] = &ps_tool_timer_stop; - start_string_functions[0] = &ps_tool_start_string; - stop_string_functions[0] = &ps_tool_stop_string; - stop_current_functions[0] = &ps_tool_stop_current; - set_parameter_functions[0] = &ps_tool_set_parameter; - dynamic_phase_start_functions[0] = &ps_tool_dynamic_phase_start; - dynamic_phase_stop_functions[0] = &ps_tool_dynamic_phase_stop; - create_counter_functions[0] = &ps_tool_create_counter; - sample_counter_functions[0] = &ps_tool_sample_counter; - set_metadata_functions[0] = &ps_tool_set_metadata; - get_timer_data_functions[0] = &ps_tool_get_timer_data; - get_counter_data_functions[0] = &ps_tool_get_counter_data; - get_metadata_functions[0] = &ps_tool_get_metadata; - free_timer_data_functions[0] = &ps_tool_free_timer_data; - free_counter_data_functions[0] = &ps_tool_free_counter_data; - free_metadata_functions[0] = &ps_tool_free_metadata; + // removing printf statement for now, it's too noisy. + //printf("Found ps_tool_initialize(), registering tool\n"); + finalize_function = &ps_tool_finalize; + pause_measurement_function = &ps_tool_pause_measurement; + resume_measurement_function = &ps_tool_resume_measurement; + register_thread_function = &ps_tool_register_thread; + dump_data_function = &ps_tool_dump_data; + timer_create_function = &ps_tool_timer_create; + timer_start_function = &ps_tool_timer_start; + timer_stop_function = &ps_tool_timer_stop; + start_string_function = &ps_tool_start_string; + stop_string_function = &ps_tool_stop_string; + stop_current_function = &ps_tool_stop_current; + set_parameter_function = &ps_tool_set_parameter; + dynamic_phase_start_function = &ps_tool_dynamic_phase_start; + dynamic_phase_stop_function = &ps_tool_dynamic_phase_stop; + create_counter_function = &ps_tool_create_counter; + sample_counter_function = &ps_tool_sample_counter; + set_metadata_function = &ps_tool_set_metadata; + get_timer_data_function = &ps_tool_get_timer_data; + get_counter_data_function = &ps_tool_get_counter_data; + get_metadata_function = &ps_tool_get_metadata; + free_timer_data_function = &ps_tool_free_timer_data; + free_counter_data_function = &ps_tool_free_counter_data; + free_metadata_function = &ps_tool_free_metadata; #else - initialize_functions[0] = + initialize_function = (ps_initialize_t)dlsym(RTLD_DEFAULT, "ps_tool_initialize"); - if (initialize_functions[0] == NULL) { + if (initialize_function == NULL) { perfstubs_initialized = PERFSTUBS_FAILURE; return; } printf("Found ps_tool_initialize(), registering tool\n"); - finalize_functions[0] = + finalize_function = (ps_finalize_t)dlsym(RTLD_DEFAULT, "ps_tool_finalize"); - pause_measurement_functions[0] = + pause_measurement_function = (ps_pause_measurement_t)dlsym(RTLD_DEFAULT, "ps_tool_pause_measurement"); - resume_measurement_functions[0] = + resume_measurement_function = (ps_resume_measurement_t)dlsym(RTLD_DEFAULT, "ps_tool_resume_measurement"); - register_thread_functions[0] = + register_thread_function = (ps_register_thread_t)dlsym(RTLD_DEFAULT, "ps_tool_register_thread"); - dump_data_functions[0] = + dump_data_function = (ps_dump_data_t)dlsym(RTLD_DEFAULT, "ps_tool_dump_data"); - timer_create_functions[0] = + timer_create_function = (ps_timer_create_t)dlsym(RTLD_DEFAULT, - "ps_tool_timer_create"); - timer_start_functions[0] = + "ps_tool_timer_create"); + timer_start_function = (ps_timer_start_t)dlsym(RTLD_DEFAULT, "ps_tool_timer_start"); - timer_stop_functions[0] = + timer_stop_function = (ps_timer_stop_t)dlsym(RTLD_DEFAULT, "ps_tool_timer_stop"); - start_string_functions[0] = + start_string_function = (ps_start_string_t)dlsym(RTLD_DEFAULT, "ps_tool_start_string"); - stop_string_functions[0] = + stop_string_function = (ps_stop_string_t)dlsym(RTLD_DEFAULT, "ps_tool_stop_string"); - stop_current_functions[0] = + stop_current_function = (ps_stop_current_t)dlsym(RTLD_DEFAULT, "ps_tool_stop_current"); - set_parameter_functions[0] = + set_parameter_function = (ps_set_parameter_t)dlsym(RTLD_DEFAULT, "ps_tool_set_parameter"); - dynamic_phase_start_functions[0] = (ps_dynamic_phase_start_t)dlsym( - RTLD_DEFAULT, "ps_tool_dynamic_phase_start"); - dynamic_phase_stop_functions[0] = (ps_dynamic_phase_stop_t)dlsym( - RTLD_DEFAULT, "ps_tool_dynamic_phase_stop"); - create_counter_functions[0] = (ps_create_counter_t)dlsym( - RTLD_DEFAULT, "ps_tool_create_counter"); - sample_counter_functions[0] = (ps_sample_counter_t)dlsym( - RTLD_DEFAULT, "ps_tool_sample_counter"); - set_metadata_functions[0] = + dynamic_phase_start_function = (ps_dynamic_phase_start_t)dlsym( + RTLD_DEFAULT, "ps_tool_dynamic_phase_start"); + dynamic_phase_stop_function = (ps_dynamic_phase_stop_t)dlsym( + RTLD_DEFAULT, "ps_tool_dynamic_phase_stop"); + create_counter_function = (ps_create_counter_t)dlsym( + RTLD_DEFAULT, "ps_tool_create_counter"); + sample_counter_function = (ps_sample_counter_t)dlsym( + RTLD_DEFAULT, "ps_tool_sample_counter"); + set_metadata_function = (ps_set_metadata_t)dlsym(RTLD_DEFAULT, "ps_tool_set_metadata"); - get_timer_data_functions[0] = (ps_get_timer_data_t)dlsym( - RTLD_DEFAULT, "ps_tool_get_timer_data"); - get_counter_data_functions[0] = (ps_get_counter_data_t)dlsym( - RTLD_DEFAULT, "ps_tool_get_counter_data"); - get_metadata_functions[0] = (ps_get_metadata_t)dlsym( - RTLD_DEFAULT, "ps_tool_get_metadata"); - free_timer_data_functions[0] = (ps_free_timer_data_t)dlsym( - RTLD_DEFAULT, "ps_tool_free_timer_data"); - free_counter_data_functions[0] = (ps_free_counter_data_t)dlsym( - RTLD_DEFAULT, "ps_tool_free_counter_data"); - free_metadata_functions[0] = (ps_free_metadata_t)dlsym( - RTLD_DEFAULT, "ps_tool_free_metadata"); + get_timer_data_function = (ps_get_timer_data_t)dlsym( + RTLD_DEFAULT, "ps_tool_get_timer_data"); + get_counter_data_function = (ps_get_counter_data_t)dlsym( + RTLD_DEFAULT, "ps_tool_get_counter_data"); + get_metadata_function = (ps_get_metadata_t)dlsym( + RTLD_DEFAULT, "ps_tool_get_metadata"); + free_timer_data_function = (ps_free_timer_data_t)dlsym( + RTLD_DEFAULT, "ps_tool_free_timer_data"); + free_counter_data_function = (ps_free_counter_data_t)dlsym( + RTLD_DEFAULT, "ps_tool_free_counter_data"); + free_metadata_function = (ps_free_metadata_t)dlsym( + RTLD_DEFAULT, "ps_tool_free_metadata"); #endif perfstubs_initialized = PERFSTUBS_SUCCESS; /* Increment the number of tools */ @@ -196,7 +195,7 @@ void initialize_library() { } char * ps_make_timer_name_(const char * file, - const char * func, int line) { + const char * func, int line) { /* The length of the line number as a string is floor(log10(abs(num))) */ int string_length = (strlen(file) + strlen(func) + floor(log10(abs(line))) + 12); char * name = calloc(string_length, sizeof(char)); @@ -206,59 +205,41 @@ char * ps_make_timer_name_(const char * file, // used internally to the class static inline void ps_register_thread_internal(void) { - //if (thread_seen == 0) { if (pthread_getspecific(key) == NULL) { - int i; - for (i = 0 ; i < num_tools_registered ; i++) { - register_thread_functions[i](); - } - //thread_seen = 1; - pthread_setspecific(key, (void*)1UL); + if (register_thread_function != NULL) { + register_thread_function(); + pthread_setspecific(key, (void*)1UL); + } } } /* Initialization */ void ps_initialize_(void) { - int i; /* Only do this once */ if (perfstubs_initialized != PERFSTUBS_UNKNOWN) { return; } initialize_library(); - for (i = 0 ; i < num_tools_registered ; i++) { - initialize_functions[i](); - } - /* No need to register the main thread */ - //thread_seen = 1; - (void) pthread_once(&key_once, make_key); - if (pthread_getspecific(key) == NULL) { - // set the key to 1, indicating we have seen this thread + if (initialize_function != NULL) { + initialize_function(); + (void) pthread_once(&key_once, make_key); pthread_setspecific(key, (void*)1UL); } } void ps_finalize_(void) { - int i; - for (i = 0 ; i < num_tools_registered ; i++) { - if (finalize_functions[i] != NULL) - finalize_functions[i](); - } + if (finalize_function != NULL) + finalize_function(); } void ps_pause_measurement_(void) { - int i; - for (i = 0 ; i < num_tools_registered ; i++) { - if (pause_measurement_functions[i] != NULL) - pause_measurement_functions[i](); - } + if (pause_measurement_function != NULL) + pause_measurement_function(); } void ps_resume_measurement_(void) { - int i; - for (i = 0 ; i < num_tools_registered ; i++) { - if (resume_measurement_functions[i] != NULL) - resume_measurement_functions[i](); - } + if (resume_measurement_function != NULL) + resume_measurement_function(); } void ps_register_thread_(void) { @@ -266,13 +247,10 @@ void ps_register_thread_(void) { } void* ps_timer_create_(const char *timer_name) { - ps_register_thread_internal(); + ps_register_thread_internal(); void ** objects = (void **)calloc(num_tools_registered, sizeof(void*)); - int i; - for (i = 0 ; i < num_tools_registered ; i++) { - if (timer_create_functions[i] != NULL) - objects[i] = (void *)timer_create_functions[i](timer_name); - } + if (timer_create_function != NULL) + objects = (void *)timer_create_function(timer_name); return (void*)(objects); } @@ -281,14 +259,10 @@ void ps_timer_create_fortran_(void ** object, const char *timer_name) { } void ps_timer_start_(void *timer) { - ps_register_thread_internal(); + ps_register_thread_internal(); void ** objects = (void **)timer; - int i; - for (i = 0; i < num_tools_registered ; i++) { - if (timer_start_functions[i] != NULL && - objects[i] != NULL) - timer_start_functions[i](objects[i]); - } + if (timer_start_function != NULL && objects != NULL) + timer_start_function(objects); } void ps_timer_start_fortran_(void **timer) { @@ -297,12 +271,9 @@ void ps_timer_start_fortran_(void **timer) { void ps_timer_stop_(void *timer) { void ** objects = (void **)timer; - int i; - for (i = 0; i < num_tools_registered ; i++) { - if (timer_stop_functions[i] != NULL && - objects[i] != NULL) - timer_stop_functions[i](objects[i]); - } + if (timer_stop_function != NULL && + objects != NULL) + timer_stop_function(objects); } void ps_timer_stop_fortran_(void **timer) { @@ -310,62 +281,41 @@ void ps_timer_stop_fortran_(void **timer) { } void ps_start_string_(const char *timer_name) { - ps_register_thread_internal(); - int i; - for (i = 0 ; i < num_tools_registered ; i++) { - if (start_string_functions[i] != NULL) - start_string_functions[i](timer_name); - } + ps_register_thread_internal(); + if (start_string_function != NULL) + start_string_function(timer_name); } void ps_stop_string_(const char *timer_name) { - int i; - for (i = 0 ; i < num_tools_registered ; i++) { - if (stop_string_functions[i] != NULL) - stop_string_functions[i](timer_name); - } + if (stop_string_function != NULL) + stop_string_function(timer_name); } void ps_stop_current_(void) { - int i; - for (i = 0 ; i < num_tools_registered ; i++) { - if (stop_current_functions[i] != NULL) - stop_current_functions[i](); - } + if (stop_current_function != NULL) + stop_current_function(); } void ps_set_parameter_(const char * parameter_name, int64_t parameter_value) { - int i; - for (i = 0; i < num_tools_registered ; i++) { - if (set_parameter_functions[i] != NULL) - set_parameter_functions[i](parameter_name, parameter_value); - } + if (set_parameter_function != NULL) + set_parameter_function(parameter_name, parameter_value); } void ps_dynamic_phase_start_(const char *phase_prefix, int iteration_index) { - int i; - for (i = 0; i < num_tools_registered ; i++) { - if (dynamic_phase_start_functions[i] != NULL) - dynamic_phase_start_functions[i](phase_prefix, iteration_index); - } + if (dynamic_phase_start_function != NULL) + dynamic_phase_start_function(phase_prefix, iteration_index); } void ps_dynamic_phase_stop_(const char *phase_prefix, int iteration_index) { - int i; - for (i = 0; i < num_tools_registered ; i++) { - if (dynamic_phase_stop_functions[i] != NULL) - dynamic_phase_stop_functions[i](phase_prefix, iteration_index); - } + if (dynamic_phase_stop_function != NULL) + dynamic_phase_stop_function(phase_prefix, iteration_index); } void* ps_create_counter_(const char *name) { - ps_register_thread_internal(); + ps_register_thread_internal(); void ** objects = (void **)calloc(num_tools_registered, sizeof(void*)); - int i; - for (i = 0 ; i < num_tools_registered ; i++) { - if (create_counter_functions[i] != NULL) - objects[i] = (void*)create_counter_functions[i](name); - } + if (create_counter_function != NULL) + objects = (void*)create_counter_function(name); return (void*)(objects); } @@ -375,12 +325,9 @@ void ps_create_counter_fortran_(void ** object, const char *name) { void ps_sample_counter_(void *counter, const double value) { void ** objects = (void **)counter; - int i; - for (i = 0; i < num_tools_registered ; i++) { - if (sample_counter_functions[i] != NULL && - objects[i] != NULL) - sample_counter_functions[i](objects[i], value); - } + if (sample_counter_function != NULL && + objects != NULL) + sample_counter_function(objects, value); } void ps_sample_counter_fortran_(void **counter, const double value) { @@ -388,61 +335,43 @@ void ps_sample_counter_fortran_(void **counter, const double value) { } void ps_set_metadata_(const char *name, const char *value) { - ps_register_thread_internal(); - int i; - for (i = 0; i < num_tools_registered ; i++) { - if (set_metadata_functions[i] != NULL) - set_metadata_functions[i](name, value); - } + ps_register_thread_internal(); + if (set_metadata_function != NULL) + set_metadata_function(name, value); } void ps_dump_data_(void) { - int i; - for (i = 0; i < num_tools_registered ; i++) { - if (dump_data_functions[i] != NULL) - dump_data_functions[i](); - } + if (dump_data_function != NULL) + dump_data_function(); } -void ps_get_timer_data_(ps_tool_timer_data_t *timer_data, int tool_id) { - if (tool_id < num_tools_registered) { - if (get_timer_data_functions[tool_id] != NULL) - get_timer_data_functions[tool_id](timer_data); - } +void ps_get_timer_data_(ps_tool_timer_data_t *timer_data) { + if (get_timer_data_function != NULL) + get_timer_data_function(timer_data); } -void ps_get_counter_data_(ps_tool_counter_data_t *counter_data, int tool_id) { - if (tool_id < num_tools_registered) { - if (get_counter_data_functions[tool_id] != NULL) - get_counter_data_functions[tool_id](counter_data); - } +void ps_get_counter_data_(ps_tool_counter_data_t *counter_data) { + if (get_counter_data_function != NULL) + get_counter_data_function(counter_data); } -void ps_get_metadata_(ps_tool_metadata_t *metadata, int tool_id) { - if (tool_id < num_tools_registered) { - if (get_metadata_functions[tool_id] != NULL) - get_metadata_functions[tool_id](metadata); - } +void ps_get_metadata_(ps_tool_metadata_t *metadata) { + if (get_metadata_function != NULL) + get_metadata_function(metadata); } -void ps_free_timer_data_(ps_tool_timer_data_t *timer_data, int tool_id) { - if (tool_id < num_tools_registered) { - if (free_timer_data_functions[tool_id] != NULL) - free_timer_data_functions[tool_id](timer_data); - } +void ps_free_timer_data_(ps_tool_timer_data_t *timer_data) { + if (free_timer_data_function != NULL) + free_timer_data_function(timer_data); } -void ps_free_counter_data_(ps_tool_counter_data_t *counter_data, int tool_id) { - if (tool_id < num_tools_registered) { - if (free_counter_data_functions[tool_id] != NULL) - free_counter_data_functions[tool_id](counter_data); - } +void ps_free_counter_data_(ps_tool_counter_data_t *counter_data) { + if (free_counter_data_function != NULL) + free_counter_data_function(counter_data); } -void ps_free_metadata_(ps_tool_metadata_t *metadata, int tool_id) { - if (tool_id < num_tools_registered) { - if (free_metadata_functions[tool_id] != NULL) - free_metadata_functions[tool_id](metadata); - } +void ps_free_metadata_(ps_tool_metadata_t *metadata) { + if (free_metadata_function != NULL) + free_metadata_function(metadata); } diff --git a/thirdparty/perfstubs/perfstubs/perfstubs_api/timer.h b/thirdparty/perfstubs/perfstubs/perfstubs_api/timer.h index c806c9ed64..6bc36978f5 100644 --- a/thirdparty/perfstubs/perfstubs/perfstubs_api/timer.h +++ b/thirdparty/perfstubs/perfstubs/perfstubs_api/timer.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2020 University of Oregon +// Copyright (c) 2019-2022 University of Oregon // Distributed under the BSD Software License // (See accompanying file LICENSE.txt) @@ -24,7 +24,8 @@ * not just the function name. If the compiler doesn't support it, * just use the function name. */ -#if defined(__GNUC__) +/* ISO C doesn't allow __PRETTY_FUNCTION__, so only do it with C++ */ +#if defined(__GNUC__) && defined(__cplusplus) #define __PERFSTUBS_FUNCTION__ __PRETTY_FUNCTION__ #else #define __PERFSTUBS_FUNCTION__ __func__ @@ -75,12 +76,12 @@ void ps_set_metadata_(const char *name, const char *value); /* data query API */ -void ps_get_timer_data_(ps_tool_timer_data_t *timer_data, int tool_id); -void ps_get_counter_data_(ps_tool_counter_data_t *counter_data, int tool_id); -void ps_get_metadata_(ps_tool_metadata_t *metadata, int tool_id); -void ps_free_timer_data_(ps_tool_timer_data_t *timer_data, int tool_id); -void ps_free_counter_data_(ps_tool_counter_data_t *counter_data, int tool_id); -void ps_free_metadata_(ps_tool_metadata_t *metadata, int tool_id); +void ps_get_timer_data_(ps_tool_timer_data_t *timer_data); +void ps_get_counter_data_(ps_tool_counter_data_t *counter_data); +void ps_get_metadata_(ps_tool_metadata_t *metadata); +void ps_free_timer_data_(ps_tool_timer_data_t *timer_data); +void ps_free_counter_data_(ps_tool_counter_data_t *counter_data); +void ps_free_metadata_(ps_tool_metadata_t *metadata); char* ps_make_timer_name_(const char * file, const char * func, int line); diff --git a/thirdparty/perfstubs/perfstubs/perfstubs_api/tool.h b/thirdparty/perfstubs/perfstubs/perfstubs_api/tool.h index d00f9ede3d..8668ffa9d8 100644 --- a/thirdparty/perfstubs/perfstubs/perfstubs_api/tool.h +++ b/thirdparty/perfstubs/perfstubs/perfstubs_api/tool.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2020 University of Oregon +// Copyright (c) 2019-2022 University of Oregon // Distributed under the BSD Software License // (See accompanying file LICENSE.txt) From 49158d015fba49f61325c9070655ac4b35d8e85c Mon Sep 17 00:00:00 2001 From: Ana Gainaru Date: Tue, 28 Nov 2023 11:25:05 -0800 Subject: [PATCH 03/18] Fix the HIP architecture CMAKE variable (#3931) --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a26c9b34b6..1a1b21d87c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -188,7 +188,7 @@ if(ADIOS2_HAVE_CUDA OR ADIOS2_HAVE_Kokkos_CUDA) endif() if(NOT DEFINED CMAKE_HIP_ARCHITECTURES AND DEFINED Kokkos_HIP_ARCHITECTURES) - set(CMAKE_HIP_ARCHITECTURES Kokkos_HIP_ARCHITECTURES) + set(CMAKE_HIP_ARCHITECTURES ${Kokkos_HIP_ARCHITECTURES}) endif() if(ADIOS2_HAVE_MPI) From 1b01e758bcb3f156f7542cf51f1d4dbaf76b8575 Mon Sep 17 00:00:00 2001 From: Ana Gainaru Date: Tue, 28 Nov 2023 14:41:01 -0500 Subject: [PATCH 04/18] Set the compiler for the Kokkos DataMan example to what is used to build Kokkos --- examples/hello/datamanKokkos/CMakeLists.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/examples/hello/datamanKokkos/CMakeLists.txt b/examples/hello/datamanKokkos/CMakeLists.txt index e0d93d9fb8..9bbe1a2a3e 100644 --- a/examples/hello/datamanKokkos/CMakeLists.txt +++ b/examples/hello/datamanKokkos/CMakeLists.txt @@ -19,7 +19,16 @@ if(NOT TARGET adios2_core) find_package(ZeroMQ 4.1 QUIET) + find_package(Kokkos 3.7 QUIET) + if(Kokkos_FOUND AND DEFINED Kokkos_CXX_COMPILER) + set(CMAKE_CXX_COMPILER "${Kokkos_CXX_COMPILER}") + endif() + find_package(ADIOS2 REQUIRED COMPONENTS ${_components}) +else() + if(DEFINED Kokkos_CXX_COMPILER) + set(CMAKE_CXX_COMPILER "${Kokkos_CXX_COMPILER}") + endif() endif() if(ADIOS2_HAVE_MPI AND ADIOS2_HAVE_DataMan) From 5ec9ef9a7e6e2a2c9d4143589206fa0e3d0ad244 Mon Sep 17 00:00:00 2001 From: Tom Gade Date: Fri, 1 Dec 2023 15:00:38 -0600 Subject: [PATCH 05/18] Fix MATLAB bindings for MacOS (#3950) * Fix MATLAB bindings for MacOS - updated bindings/Matlab/Makefile, added preprocessor macro to check MPI usage and call the correct version of adios2_init() accordingly, fixed spelling error in bindings/C/adios2/c/adios2_c_adios.h * Update adiosopenc.c to use adios2_init_serial regardless of MPI use. --- bindings/C/adios2/c/adios2_c_adios.h | 2 +- bindings/Matlab/Makefile | 10 ++++++++++ bindings/Matlab/adiosopenc.c | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/bindings/C/adios2/c/adios2_c_adios.h b/bindings/C/adios2/c/adios2_c_adios.h index b5044b0447..147cb8ab6a 100644 --- a/bindings/C/adios2/c/adios2_c_adios.h +++ b/bindings/C/adios2/c/adios2_c_adios.h @@ -46,7 +46,7 @@ adios2_adios *adios2_init_config_mpi(const char *config_file, MPI_Comm comm); #else #define adios2_init() adios2_init_serial() -#define adios2_init_config(config_file) adios2_init_config_seria(config_file) +#define adios2_init_config(config_file) adios2_init_config_serial(config_file) #endif /** diff --git a/bindings/Matlab/Makefile b/bindings/Matlab/Makefile index 09978c32b8..aaaebf3a0e 100644 --- a/bindings/Matlab/Makefile +++ b/bindings/Matlab/Makefile @@ -18,6 +18,16 @@ MEXLIBS="LDFLAGS=${ADIOS_LIBS}" ADIOS_INC=-I${ADIOS_DIR}/include ADIOS_LIBS=`${ADIOS_DIR}/bin/adios2-config --c-libs` +### MacOS - example using homebrew installed ADIOS2 and Xcode 15 clang +### 1) Install homebrew (https://brew.sh/) and Xcode (App Store) +### 2) brew install adios2 +### OR +### 2) Compile Adios2 from scratch and update ADIOS_DIR below to match install directory +#ADIOS_DIR=/opt/homebrew/opt/adios2 +#ADIOS_INC=-I${ADIOS_DIR}/include +#ADIOS_LIBS=-Wl,-rpath,${ADIOS_DIR}/lib -shared -L${ADIOS_DIR}/lib -ladios2_c -ladios2_core +#MEXLIBS="LDFLAGS=${ADIOS_LIBS}" + MEXOPTS=-largeArrayDims -DDEBUG CFLAGS="-g -std=c99 -fPIC -O0" default: diff --git a/bindings/Matlab/adiosopenc.c b/bindings/Matlab/adiosopenc.c index 95dc6d41b3..b9b4c1a88a 100644 --- a/bindings/Matlab/adiosopenc.c +++ b/bindings/Matlab/adiosopenc.c @@ -140,7 +140,7 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]) /********************************************************/ /* Open ADIOS file now and get variables and attributes */ - adiosobj = adios2_init(false); + adiosobj = adios2_init_serial(); group = adios2_declare_io(adiosobj, "matlabiogroup"); // name is arbitrary fp = adios2_open(group, fname, adios2_mode_read); if (fp == NULL) From a0361bff4d5ce8d2cbfcdfbbda3fd98ed9d1434e Mon Sep 17 00:00:00 2001 From: Spiros Tsalikis Date: Fri, 1 Dec 2023 12:57:47 -0500 Subject: [PATCH 06/18] Delete VTK code in examples --- .../gray-scott-struct/CMakeLists.txt | 19 - .../gray-scott-struct/analysis/curvature.cpp | 202 ---------- .../gray-scott-struct/analysis/find_blobs.cpp | 245 ------------- .../gray-scott-struct/analysis/isosurface.cpp | 346 ------------------ .../plot/render_isosurface.cpp | 204 ----------- .../simulations/gray-scott/CMakeLists.txt | 18 - .../gray-scott/analysis/curvature.cpp | 202 ---------- .../gray-scott/analysis/find_blobs.cpp | 245 ------------- .../gray-scott/analysis/isosurface.cpp | 346 ------------------ .../gray-scott/plot/render_isosurface.cpp | 204 ----------- 10 files changed, 2031 deletions(-) delete mode 100644 examples/simulations/gray-scott-struct/analysis/curvature.cpp delete mode 100644 examples/simulations/gray-scott-struct/analysis/find_blobs.cpp delete mode 100644 examples/simulations/gray-scott-struct/analysis/isosurface.cpp delete mode 100644 examples/simulations/gray-scott-struct/plot/render_isosurface.cpp delete mode 100644 examples/simulations/gray-scott/analysis/curvature.cpp delete mode 100644 examples/simulations/gray-scott/analysis/find_blobs.cpp delete mode 100644 examples/simulations/gray-scott/analysis/isosurface.cpp delete mode 100644 examples/simulations/gray-scott/plot/render_isosurface.cpp diff --git a/examples/simulations/gray-scott-struct/CMakeLists.txt b/examples/simulations/gray-scott-struct/CMakeLists.txt index 8538e6bd07..5d67c49447 100644 --- a/examples/simulations/gray-scott-struct/CMakeLists.txt +++ b/examples/simulations/gray-scott-struct/CMakeLists.txt @@ -48,23 +48,4 @@ if(ADIOS2_HAVE_MPI) install(DIRECTORY "catalyst" DESTINATION ${CMAKE_INSTALL_PREFIX}/share/adios2/gray-scott-struct) install(PROGRAMS "cleanup.sh" DESTINATION ${CMAKE_INSTALL_PREFIX}/share/adios2/gray-scott-struct) - - find_package(VTK QUIET) - if(VTK_FOUND) - add_executable(adios2_simulations_gray-scott-struct_curvature analysis/curvature.cpp) - target_link_libraries(adios2_simulations_gray-scott-struct_curvature adios2::cxx11_mpi MPI::MPI_C ${VTK_LIBRARIES}) - install(TARGETS adios2_simulations_gray-scott-struct_curvature RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) - - add_executable(adios2_simulations_gray-scott-struct_find_blobs analysis/find_blobs.cpp) - target_link_libraries(adios2_simulations_gray-scott-struct_find_blobs adios2::cxx11_mpi MPI::MPI_C ${VTK_LIBRARIES}) - install(TARGETS adios2_simulations_gray-scott-struct_find_blobs RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) - - add_executable(adios2_simulations_gray-scott-struct_isosurface analysis/isosurface.cpp) - target_link_libraries(adios2_simulations_gray-scott-struct_isosurface adios2::cxx11_mpi MPI::MPI_C ${VTK_LIBRARIES}) - install(TARGETS adios2_simulations_gray-scott-struct_isosurface RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) - - # add_executable(adios2_simulations_gray-scott-struct_render_isosurface plot/render_isosurface.cpp) - # target_link_libraries(adios2_simulations_gray-scott_struct_render-isosurface adios2::cxx11_mpi MPI::MPI_C ${VTK_LIBRARIES}) - # install(TARGETS adios2_simulations_gray-scott-struct_render_isosurface RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) - endif() endif() diff --git a/examples/simulations/gray-scott-struct/analysis/curvature.cpp b/examples/simulations/gray-scott-struct/analysis/curvature.cpp deleted file mode 100644 index 586e2fe7d2..0000000000 --- a/examples/simulations/gray-scott-struct/analysis/curvature.cpp +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * Analysis code for the Gray-Scott simulation. - * Computes mean curvature at each point on an isosurface. - * - * Keichi Takahashi - * - */ - -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../common/timer.hpp" - -vtkSmartPointer read_mesh(const std::vector &bufPoints, - const std::vector &bufCells, - const std::vector &bufNormals) -{ - int nPoints = static_cast(bufPoints.size() / 3); - int nCells = static_cast(bufCells.size() / 3); - - auto points = vtkSmartPointer::New(); - points->SetNumberOfPoints(nPoints); - for (vtkIdType i = 0; i < nPoints; i++) - { - points->SetPoint(i, &bufPoints[i * 3]); - } - - auto polys = vtkSmartPointer::New(); - for (vtkIdType i = 0; i < nCells; i++) - { - vtkIdType a = bufCells[i * 3 + 0]; - vtkIdType b = bufCells[i * 3 + 1]; - vtkIdType c = bufCells[i * 3 + 2]; - - polys->InsertNextCell(3); - polys->InsertCellPoint(a); - polys->InsertCellPoint(b); - polys->InsertCellPoint(c); - } - - auto normals = vtkSmartPointer::New(); - normals->SetNumberOfComponents(3); - for (vtkIdType i = 0; i < nPoints; i++) - { - normals->InsertNextTuple(&bufNormals[i * 3]); - } - - auto polyData = vtkSmartPointer::New(); - polyData->SetPoints(points); - polyData->SetPolys(polys); - polyData->GetPointData()->SetNormals(normals); - - return polyData; -} - -void compute_curvature(const vtkSmartPointer polyData) -{ - vtkSmartPointer curvaturesFilter = vtkSmartPointer::New(); - curvaturesFilter->SetInputData(polyData); - // curvaturesFilter->SetCurvatureTypeToMinimum(); - // curvaturesFilter->SetCurvatureTypeToMaximum(); - // curvaturesFilter->SetCurvatureTypeToGaussian(); - curvaturesFilter->SetCurvatureTypeToMean(); - curvaturesFilter->Update(); -} - -int main(int argc, char *argv[]) -{ - int provided; - MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); - - int rank, procs, wrank; - - MPI_Comm_rank(MPI_COMM_WORLD, &wrank); - - const unsigned int color = 7; - MPI_Comm comm; - MPI_Comm_split(MPI_COMM_WORLD, color, wrank, &comm); - - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &procs); - - if (argc < 3) - { - if (rank == 0) - { - std::cerr << "Too few arguments" << std::endl; - std::cout << "Usage: curvature input output" << std::endl; - } - MPI_Abort(MPI_COMM_WORLD, -1); - } - - const std::string input_fname(argv[1]); - const std::string output_fname(argv[2]); - - adios2::ADIOS adios("adios2.xml", comm); - - adios2::IO inIO = adios.DeclareIO("IsosurfaceOutput"); - adios2::Engine reader = inIO.Open(input_fname, adios2::Mode::Read); - - adios2::IO outIO = adios.DeclareIO("CurvatureOutput"); - adios2::Engine writer = outIO.Open(output_fname, adios2::Mode::Write); - - std::vector points; - std::vector cells; - std::vector normals; - int step; - -#ifdef ENABLE_TIMERS - Timer timer_total; - Timer timer_read; - Timer timer_compute; - - std::ostringstream log_fname; - log_fname << "compute_curvature_pe_" << rank << ".log"; - - std::ofstream log(log_fname.str()); - log << "step\ttotal_curv\tread_curv\tcompute_curv" << std::endl; -#endif - - while (true) - { -#ifdef ENABLE_TIMERS - MPI_Barrier(comm); - timer_total.start(); - timer_read.start(); -#endif - - adios2::StepStatus status = reader.BeginStep(); - - if (status != adios2::StepStatus::OK) - { - break; - } - - auto varPoint = inIO.InquireVariable("point"); - auto varCell = inIO.InquireVariable("cell"); - auto varNormal = inIO.InquireVariable("normal"); - auto varStep = inIO.InquireVariable("step"); - - if (varPoint.Shape().size() > 0 || varCell.Shape().size() > 0) - { - varPoint.SetSelection({{0, 0}, {varPoint.Shape()[0], varPoint.Shape()[1]}}); - varCell.SetSelection({{0, 0}, {varCell.Shape()[0], varCell.Shape()[1]}}); - varNormal.SetSelection({{0, 0}, {varNormal.Shape()[0], varNormal.Shape()[1]}}); - - reader.Get(varPoint, points); - reader.Get(varCell, cells); - reader.Get(varNormal, normals); - } - - reader.Get(varStep, &step); - - reader.EndStep(); - -#ifdef ENABLE_TIMERS - double time_read = timer_read.stop(); - MPI_Barrier(comm); - timer_compute.start(); -#endif - - vtkSmartPointer polyData = read_mesh(points, cells, normals); - compute_curvature(polyData); - - if (!rank) - { - std::cout << "compute_curvature at step " << step << std::endl; - } - -#ifdef ENABLE_TIMERS - double time_compute = timer_compute.stop(); - double time_step = timer_total.stop(); - MPI_Barrier(comm); - - log << step << "\t" << time_step << "\t" << time_read << "\t" << time_compute << std::endl; -#endif - } - -#ifdef ENABLE_TIMERS - log << "total\t" << timer_total.elapsed() << "\t" << timer_read.elapsed() << "\t" - << timer_compute.elapsed() << "\t" << std::endl; - - log.close(); -#endif - - writer.Close(); - reader.Close(); -} diff --git a/examples/simulations/gray-scott-struct/analysis/find_blobs.cpp b/examples/simulations/gray-scott-struct/analysis/find_blobs.cpp deleted file mode 100644 index 8b5c8c54b2..0000000000 --- a/examples/simulations/gray-scott-struct/analysis/find_blobs.cpp +++ /dev/null @@ -1,245 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * Analysis code for the Gray-Scott simulation. - * Reads iso-surface mesh data and detects connected components. - * Counts the total number of connected components and measures the surface - * area of each component. - * - * Keichi Takahashi - * - */ - -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../common/timer.hpp" - -vtkSmartPointer read_mesh(const std::vector &bufPoints, - const std::vector &bufCells, - const std::vector &bufNormals) -{ - int nPoints = static_cast(bufPoints.size() / 3); - int nCells = static_cast(bufCells.size() / 3); - - auto points = vtkSmartPointer::New(); - points->SetNumberOfPoints(nPoints); - for (vtkIdType i = 0; i < nPoints; i++) - { - points->SetPoint(i, &bufPoints[i * 3]); - } - - auto polys = vtkSmartPointer::New(); - for (vtkIdType i = 0; i < nCells; i++) - { - vtkIdType a = bufCells[i * 3 + 0]; - vtkIdType b = bufCells[i * 3 + 1]; - vtkIdType c = bufCells[i * 3 + 2]; - - polys->InsertNextCell(3); - polys->InsertCellPoint(a); - polys->InsertCellPoint(b); - polys->InsertCellPoint(c); - } - - auto normals = vtkSmartPointer::New(); - normals->SetNumberOfComponents(3); - for (vtkIdType i = 0; i < nPoints; i++) - { - normals->InsertNextTuple(&bufNormals[i * 3]); - } - - auto polyData = vtkSmartPointer::New(); - polyData->SetPoints(points); - polyData->SetPolys(polys); - polyData->GetPointData()->SetNormals(normals); - - return polyData; -} - -void find_blobs(const vtkSmartPointer polyData) -{ - auto connectivityFilter = vtkSmartPointer::New(); - connectivityFilter->SetInputData(polyData); - connectivityFilter->SetExtractionModeToAllRegions(); - connectivityFilter->ColorRegionsOn(); - connectivityFilter->Update(); - - int nBlobs = connectivityFilter->GetNumberOfExtractedRegions(); - - std::cout << "Found " << nBlobs << " blobs" << std::endl; - - auto threshold = vtkSmartPointer::New(); - auto massProperties = vtkSmartPointer::New(); - auto surfaceFilter = vtkSmartPointer::New(); - - threshold->SetInputConnection(connectivityFilter->GetOutputPort()); - surfaceFilter->SetInputConnection(threshold->GetOutputPort()); - massProperties->SetInputConnection(surfaceFilter->GetOutputPort()); - - for (int i = 0; i < nBlobs; i++) - { - threshold->SetThresholdFunction(vtkThreshold::THRESHOLD_BETWEEN); - threshold->SetLowerThreshold(i); - threshold->SetUpperThreshold(i); - - std::cout << "Surface area of blob #" << i << " is " << massProperties->GetSurfaceArea() - << std::endl; - } -} - -void find_largest_blob(const vtkSmartPointer polyData) -{ - auto connectivityFilter = vtkSmartPointer::New(); - connectivityFilter->SetInputData(polyData); - connectivityFilter->SetExtractionModeToLargestRegion(); - connectivityFilter->Update(); - - auto massProperties = vtkSmartPointer::New(); - auto surfaceFilter = vtkSmartPointer::New(); - - surfaceFilter->SetInputConnection(connectivityFilter->GetOutputPort()); - massProperties->SetInputConnection(surfaceFilter->GetOutputPort()); - - std::cout << "Surface area of largest blob is " << massProperties->GetSurfaceArea() - << std::endl; -} - -int main(int argc, char *argv[]) -{ - int provided; - MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); - - int rank, procs, wrank; - - MPI_Comm_rank(MPI_COMM_WORLD, &wrank); - - const unsigned int color = 6; - MPI_Comm comm; - MPI_Comm_split(MPI_COMM_WORLD, color, wrank, &comm); - - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &procs); - - if (argc < 2) - { - if (rank == 0) - { - std::cerr << "Too few arguments" << std::endl; - std::cout << "Usage: find_blobs input" << std::endl; - } - MPI_Abort(MPI_COMM_WORLD, -1); - } - - if (procs != 1) - { - if (rank == 0) - { - std::cerr << "find_blobs only supports serial execution" << std::endl; - } - MPI_Abort(MPI_COMM_WORLD, -1); - } - - const std::string input_fname(argv[1]); - - adios2::ADIOS adios("adios2.xml", comm); - - adios2::IO inIO = adios.DeclareIO("IsosurfaceOutput"); - adios2::Engine reader = inIO.Open(input_fname, adios2::Mode::Read); - - std::vector points; - std::vector cells; - std::vector normals; - int step; - -#ifdef ENABLE_TIMERS - Timer timer_total; - Timer timer_compute; - Timer timer_read; - - std::ostringstream log_fname; - log_fname << "find_blobs_pe_" << rank << ".log"; - - std::ofstream log(log_fname.str()); - log << "step\ttotal_blobs\tread_blobs\tcompute_blobs" << std::endl; -#endif - - while (true) - { -#ifdef ENABLE_TIMERS - MPI_Barrier(comm); - timer_total.start(); - timer_read.start(); -#endif - - adios2::StepStatus status = reader.BeginStep(); - - if (status != adios2::StepStatus::OK) - { - break; - } - - auto varPoint = inIO.InquireVariable("point"); - auto varCell = inIO.InquireVariable("cell"); - auto varNormal = inIO.InquireVariable("normal"); - auto varStep = inIO.InquireVariable("step"); - - if (varPoint.Shape().size() > 0 || varCell.Shape().size() > 0) - { - varPoint.SetSelection({{0, 0}, {varPoint.Shape()[0], varPoint.Shape()[1]}}); - varCell.SetSelection({{0, 0}, {varCell.Shape()[0], varCell.Shape()[1]}}); - varNormal.SetSelection({{0, 0}, {varNormal.Shape()[0], varNormal.Shape()[1]}}); - - reader.Get(varPoint, points); - reader.Get(varCell, cells); - reader.Get(varNormal, normals); - } - - reader.Get(varStep, &step); - - reader.EndStep(); - -#ifdef ENABLE_TIMERS - double time_read = timer_read.stop(); - MPI_Barrier(comm); - timer_compute.start(); -#endif - - std::cout << "find_blobs at step " << step << std::endl; - - auto polyData = read_mesh(points, cells, normals); - // find_blobs(polyData); - find_largest_blob(polyData); - -#ifdef ENABLE_TIMERS - double time_compute = timer_compute.stop(); - double time_step = timer_total.stop(); - MPI_Barrier(comm); - - log << step << "\t" << time_step << "\t" << time_read << "\t" << time_compute << std::endl; -#endif - } - -#ifdef ENABLE_TIMERS - log << "total\t" << timer_total.elapsed() << "\t" << timer_read.elapsed() << "\t" - << timer_compute.elapsed() << std::endl; - - log.close(); -#endif - - reader.Close(); -} diff --git a/examples/simulations/gray-scott-struct/analysis/isosurface.cpp b/examples/simulations/gray-scott-struct/analysis/isosurface.cpp deleted file mode 100644 index e02a507aaa..0000000000 --- a/examples/simulations/gray-scott-struct/analysis/isosurface.cpp +++ /dev/null @@ -1,346 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * Analysis code for the Gray-Scott simulation. - * Reads variable U and and extracts the iso-surface using VTK. - * Writes the extracted iso-surface using ADIOS. - * - * Keichi Takahashi - * - */ - -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../common/timer.hpp" - -vtkSmartPointer compute_isosurface(const adios2::Variable &varField, - const std::vector &field, double isovalue) -{ - // Convert field values to vtkImageData - auto importer = vtkSmartPointer::New(); - importer->SetDataSpacing(1, 1, 1); - importer->SetDataOrigin(static_cast(varField.Start()[2]), - static_cast(varField.Start()[1]), - static_cast(varField.Start()[0])); - importer->SetWholeExtent(0, static_cast(varField.Count()[2] - 1), 0, - static_cast(varField.Count()[1] - 1), 0, - static_cast(varField.Count()[0] - 1)); - importer->SetDataExtentToWholeExtent(); - importer->SetDataScalarTypeToDouble(); - importer->SetNumberOfScalarComponents(1); - importer->SetImportVoidPointer(const_cast(field.data())); - - // Run the marching cubes algorithm - auto mcubes = vtkSmartPointer::New(); - mcubes->SetInputConnection(importer->GetOutputPort()); - mcubes->ComputeNormalsOn(); - mcubes->SetValue(0, isovalue); - mcubes->Update(); - - // Return the isosurface as vtkPolyData - return mcubes->GetOutput(); -} - -void write_vtk(const std::string &fname, const vtkSmartPointer polyData) -{ - auto writer = vtkSmartPointer::New(); - writer->SetFileName(fname.c_str()); - writer->SetInputData(polyData); - writer->Write(); -} - -void write_adios(adios2::Engine &writer, const vtkSmartPointer polyData, - adios2::Variable &varPoint, adios2::Variable &varCell, - adios2::Variable &varNormal, adios2::Variable &varOutStep, int step, - MPI_Comm comm) -{ - int numCells = static_cast(polyData->GetNumberOfPolys()); - int numPoints = static_cast(polyData->GetNumberOfPoints()); - int rank; - - MPI_Comm_rank(comm, &rank); - - std::vector points(static_cast(numPoints * 3)); - std::vector normals(static_cast(numPoints * 3)); - std::vector cells(static_cast(numCells * 3)); // Assumes that cells are triangles - - double coords[3]; - - auto cellArray = polyData->GetPolys(); - - cellArray->InitTraversal(); - - // Iterate through cells - for (vtkIdType i = 0; i < polyData->GetNumberOfPolys(); i++) - { - auto idList = vtkSmartPointer::New(); - - cellArray->GetNextCell(idList); - - // Iterate through points of a cell - for (vtkIdType j = 0; j < idList->GetNumberOfIds(); j++) - { - auto id = idList->GetId(j); - - cells[i * 3 + j] = static_cast(id); - - polyData->GetPoint(id, coords); - - points[id * 3 + 0] = coords[0]; - points[id * 3 + 1] = coords[1]; - points[id * 3 + 2] = coords[2]; - } - } - - auto normalArray = polyData->GetPointData()->GetNormals(); - - // Extract normals - for (int i = 0; i < normalArray->GetNumberOfTuples(); i++) - { - normalArray->GetTuple(i, coords); - - normals[i * 3 + 0] = coords[0]; - normals[i * 3 + 1] = coords[1]; - normals[i * 3 + 2] = coords[2]; - } - - int totalPoints, offsetPoints; - MPI_Allreduce(&numPoints, &totalPoints, 1, MPI_INT, MPI_SUM, comm); - MPI_Scan(&numPoints, &offsetPoints, 1, MPI_INT, MPI_SUM, comm); - - writer.BeginStep(); - - varPoint.SetShape( - {static_cast(totalPoints), static_cast(totalPoints > 0 ? 3 : 0)}); - varPoint.SetSelection( - {{static_cast(offsetPoints - numPoints), 0}, - {static_cast(numPoints), static_cast(numPoints > 0 ? 3 : 0)}}); - - varNormal.SetShape(varPoint.Shape()); - varNormal.SetSelection({varPoint.Start(), varPoint.Count()}); - - if (numPoints) - { - writer.Put(varPoint, points.data()); - writer.Put(varNormal, normals.data()); - } - - int totalCells, offsetCells; - MPI_Allreduce(&numCells, &totalCells, 1, MPI_INT, MPI_SUM, comm); - MPI_Scan(&numCells, &offsetCells, 1, MPI_INT, MPI_SUM, comm); - - for (int &cell : cells) - { - cell += (offsetPoints - numPoints); - } - - varCell.SetShape( - {static_cast(totalCells), static_cast(totalCells > 0 ? 3 : 0)}); - varCell.SetSelection( - {{static_cast(offsetCells - numCells), 0}, - {static_cast(numCells), static_cast(numCells > 0 ? 3 : 0)}}); - - if (numCells) - { - writer.Put(varCell, cells.data()); - } - - if (!rank) - { - std::cout << "isosurface at step " << step << " writing out " << totalCells << " cells and " - << totalPoints << " points" << std::endl; - } - - writer.Put(varOutStep, step); - - writer.EndStep(); -} - -int main(int argc, char *argv[]) -{ - int provided; - MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); - - int rank, procs, wrank; - - MPI_Comm_rank(MPI_COMM_WORLD, &wrank); - - const unsigned int color = 5; - MPI_Comm comm; - MPI_Comm_split(MPI_COMM_WORLD, color, wrank, &comm); - - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &procs); - - int dims[3] = {0}; - MPI_Dims_create(procs, 3, dims); - size_t npx = dims[0]; - size_t npy = dims[1]; - size_t npz = dims[2]; - - int coords[3] = {0}; - int periods[3] = {0}; - MPI_Comm cart_comm; - MPI_Cart_create(comm, 3, dims, periods, 0, &cart_comm); - MPI_Cart_coords(cart_comm, rank, 3, coords); - size_t px = coords[0]; - size_t py = coords[1]; - size_t pz = coords[2]; - - if (argc < 4) - { - if (rank == 0) - { - std::cerr << "Too few arguments" << std::endl; - std::cout << "Usage: isosurface input output isovalues..." << std::endl; - } - MPI_Abort(MPI_COMM_WORLD, -1); - } - - const std::string input_fname(argv[1]); - const std::string output_fname(argv[2]); - - std::vector isovalues; - for (int i = 3; i < argc; i++) - { - isovalues.push_back(std::stod(argv[i])); - } - - adios2::ADIOS adios("adios2.xml", comm); - - adios2::IO inIO = adios.DeclareIO("SimulationOutput"); - adios2::Engine reader = inIO.Open(input_fname, adios2::Mode::Read); - - adios2::IO outIO = adios.DeclareIO("IsosurfaceOutput"); - adios2::Engine writer = outIO.Open(output_fname, adios2::Mode::Write); - - auto varPoint = outIO.DefineVariable("point", {1, 3}, {0, 0}, {1, 3}); - auto varCell = outIO.DefineVariable("cell", {1, 3}, {0, 0}, {1, 3}); - auto varNormal = outIO.DefineVariable("normal", {1, 3}, {0, 0}, {1, 3}); - auto varOutStep = outIO.DefineVariable("step"); - - std::vector u; - int step; - -#ifdef ENABLE_TIMERS - Timer timer_total; - Timer timer_read; - Timer timer_compute; - Timer timer_write; - - std::ostringstream log_fname; - log_fname << "isosurface_pe_" << rank << ".log"; - - std::ofstream log(log_fname.str()); - log << "step\ttotal_iso\tread_iso\tcompute_iso\twrite_iso" << std::endl; -#endif - - while (true) - { -#ifdef ENABLE_TIMERS - MPI_Barrier(comm); - timer_total.start(); - timer_read.start(); -#endif - - adios2::StepStatus status = reader.BeginStep(); - - if (status != adios2::StepStatus::OK) - { - break; - } - - adios2::Variable varU = inIO.InquireVariable("U"); - const adios2::Variable varStep = inIO.InquireVariable("step"); - - adios2::Dims shape = varU.Shape(); - - size_t size_x = (shape[0] + npx - 1) / npx; - size_t size_y = (shape[1] + npy - 1) / npy; - size_t size_z = (shape[2] + npz - 1) / npz; - - size_t offset_x = size_x * px; - size_t offset_y = size_y * py; - size_t offset_z = size_z * pz; - - if (px == npx - 1) - { - size_x -= size_x * npx - shape[0]; - } - if (py == npy - 1) - { - size_y -= size_y * npy - shape[1]; - } - if (pz == npz - 1) - { - size_z -= size_z * npz - shape[2]; - } - - varU.SetSelection({{offset_x, offset_y, offset_z}, - {size_x + (px != npx - 1 ? 1 : 0), size_y + (py != npy - 1 ? 1 : 0), - size_z + (pz != npz - 1 ? 1 : 0)}}); - - reader.Get(varU, u); - reader.Get(varStep, step); - reader.EndStep(); - -#ifdef ENABLE_TIMERS - double time_read = timer_read.stop(); - MPI_Barrier(comm); - timer_compute.start(); -#endif - - auto appendFilter = vtkSmartPointer::New(); - - for (const auto isovalue : isovalues) - { - auto polyData = compute_isosurface(varU, u, isovalue); - appendFilter->AddInputData(polyData); - } - - appendFilter->Update(); - -#ifdef ENABLE_TIMERS - double time_compute = timer_compute.stop(); - MPI_Barrier(comm); - timer_write.start(); -#endif - - write_adios(writer, appendFilter->GetOutput(), varPoint, varCell, varNormal, varOutStep, - step, comm); - -#ifdef ENABLE_TIMERS - double time_write = timer_write.stop(); - double time_step = timer_total.stop(); - MPI_Barrier(comm); - - log << step << "\t" << time_step << "\t" << time_read << "\t" << time_compute << "\t" - << time_write << std::endl; -#endif - } - -#ifdef ENABLE_TIMERS - log << "total\t" << timer_total.elapsed() << "\t" << timer_read.elapsed() << "\t" - << timer_compute.elapsed() << "\t" << timer_write.elapsed() << std::endl; - - log.close(); -#endif - - writer.Close(); - reader.Close(); - - MPI_Finalize(); -} diff --git a/examples/simulations/gray-scott-struct/plot/render_isosurface.cpp b/examples/simulations/gray-scott-struct/plot/render_isosurface.cpp deleted file mode 100644 index 30a464b831..0000000000 --- a/examples/simulations/gray-scott-struct/plot/render_isosurface.cpp +++ /dev/null @@ -1,204 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * Visualization code for the Gray-Scott simulation. - * Reads and renders iso-surface mesh data. - * - * Keichi Takahashi - * - */ - -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -VTK_MODULE_INIT(vtkRenderingOpenGL2); - -typedef struct -{ - vtkRenderView *renderView; - vtkPolyDataMapper *mapper; - adios2::IO *inIO; - adios2::Engine *reader; -} Context; - -vtkSmartPointer read_mesh(const std::vector &bufPoints, - const std::vector &bufCells, - const std::vector &bufNormals) -{ - int nPoints = bufPoints.size() / 3; - int nCells = bufCells.size() / 3; - - auto points = vtkSmartPointer::New(); - points->SetNumberOfPoints(nPoints); - for (vtkIdType i = 0; i < nPoints; i++) - { - points->SetPoint(i, &bufPoints[i * 3]); - } - - auto polys = vtkSmartPointer::New(); - for (vtkIdType i = 0; i < nCells; i++) - { - vtkIdType a = bufCells[i * 3 + 0]; - vtkIdType b = bufCells[i * 3 + 1]; - vtkIdType c = bufCells[i * 3 + 2]; - - polys->InsertNextCell(3); - polys->InsertCellPoint(a); - polys->InsertCellPoint(b); - polys->InsertCellPoint(c); - } - - auto normals = vtkSmartPointer::New(); - normals->SetNumberOfComponents(3); - for (vtkIdType i = 0; i < nPoints; i++) - { - normals->InsertNextTuple(&bufNormals[i * 3]); - } - - auto polyData = vtkSmartPointer::New(); - polyData->SetPoints(points); - polyData->SetPolys(polys); - polyData->GetPointData()->SetNormals(normals); - - return polyData; -} - -void timer_func(vtkObject *object, unsigned long eid, void *clientdata, void *calldata) -{ - Context *context = static_cast(clientdata); - - std::vector points; - std::vector cells; - std::vector normals; - int step; - - adios2::StepStatus status = context->reader->BeginStep(); - - if (status != adios2::StepStatus::OK) - { - return; - } - - auto varPoint = context->inIO->InquireVariable("point"); - auto varCell = context->inIO->InquireVariable("cell"); - auto varNormal = context->inIO->InquireVariable("normal"); - auto varStep = context->inIO->InquireVariable("step"); - - if (varPoint.Shape().size() > 0 || varCell.Shape().size() > 0) - { - varPoint.SetSelection({{0, 0}, {varPoint.Shape()[0], varPoint.Shape()[1]}}); - varCell.SetSelection({{0, 0}, {varCell.Shape()[0], varCell.Shape()[1]}}); - varNormal.SetSelection({{0, 0}, {varNormal.Shape()[0], varNormal.Shape()[1]}}); - - context->reader->Get(varPoint, points); - context->reader->Get(varCell, cells); - context->reader->Get(varNormal, normals); - } - - context->reader->Get(varStep, &step); - - context->reader->EndStep(); - - std::cout << "render_isosurface at step " << step << std::endl; - - vtkSmartPointer polyData = read_mesh(points, cells, normals); - - context->mapper->SetInputData(polyData); - context->renderView->ResetCamera(); - context->renderView->Render(); -} - -int main(int argc, char *argv[]) -{ - MPI_Init(&argc, &argv); - - int rank, procs, wrank; - - MPI_Comm_rank(MPI_COMM_WORLD, &wrank); - - const unsigned int color = 7; - MPI_Comm comm; - MPI_Comm_split(MPI_COMM_WORLD, color, wrank, &comm); - - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &procs); - - if (argc < 2) - { - if (rank == 0) - { - std::cerr << "Too few arguments" << std::endl; - std::cout << "Usage: render_isosurface input" << std::endl; - } - MPI_Abort(MPI_COMM_WORLD, -1); - } - - if (procs != 1) - { - if (rank == 0) - { - std::cerr << "render_isosurface only supports serial execution" << std::endl; - } - MPI_Abort(MPI_COMM_WORLD, -1); - } - - const std::string input_fname(argv[1]); - - adios2::ADIOS adios("adios2.xml", comm); - - adios2::IO inIO = adios.DeclareIO("IsosurfaceOutput"); - adios2::Engine reader = inIO.Open(input_fname, adios2::Mode::Read); - - auto mapper = vtkSmartPointer::New(); - - auto actor = vtkSmartPointer::New(); - actor->SetMapper(mapper); - - auto renderView = vtkSmartPointer::New(); - renderView->GetRenderer()->AddActor(actor); - renderView->Update(); - - auto style = vtkSmartPointer::New(); - style->SetCurrentStyleToTrackballCamera(); - - auto interactor = renderView->GetInteractor(); - interactor->Initialize(); - interactor->SetInteractorStyle(style); - interactor->CreateRepeatingTimer(100); - - Context context = { - .renderView = renderView, - .mapper = mapper, - .inIO = &inIO, - .reader = &reader, - }; - - auto timerCallback = vtkSmartPointer::New(); - timerCallback->SetCallback(timer_func); - timerCallback->SetClientData(&context); - interactor->AddObserver(vtkCommand::TimerEvent, timerCallback); - - renderView->Render(); - interactor->Start(); - - reader.Close(); -} diff --git a/examples/simulations/gray-scott/CMakeLists.txt b/examples/simulations/gray-scott/CMakeLists.txt index e58acd4144..c39444698a 100644 --- a/examples/simulations/gray-scott/CMakeLists.txt +++ b/examples/simulations/gray-scott/CMakeLists.txt @@ -48,22 +48,4 @@ if(ADIOS2_HAVE_MPI) install(DIRECTORY "catalyst" DESTINATION ${CMAKE_INSTALL_PREFIX}/share/adios2/gray-scott) install(PROGRAMS "cleanup.sh" DESTINATION ${CMAKE_INSTALL_PREFIX}/share/adios2/gray-scott) - - find_package(VTK QUIET) - if(VTK_FOUND) - add_executable(adios2_simulations_gray-scott_curvature analysis/curvature.cpp) - target_link_libraries(adios2_simulations_gray-scott_curvature adios2::cxx11_mpi MPI::MPI_C ${VTK_LIBRARIES}) - install(TARGETS adios2_simulations_gray-scott_curvature RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) - - add_executable(adios2_simulations_gray-scott_find-blobs analysis/find_blobs.cpp) - target_link_libraries(adios2_simulations_gray-scott_find-blobs adios2::cxx11_mpi MPI::MPI_C ${VTK_LIBRARIES}) - install(TARGETS adios2_simulations_gray-scott_find-blobs RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) - - add_executable(adios2_simulations_gray-scott_isosurface analysis/isosurface.cpp) - target_link_libraries(adios2_simulations_gray-scott_isosurface adios2::cxx11_mpi MPI::MPI_C ${VTK_LIBRARIES}) - install(TARGETS adios2_simulations_gray-scott_isosurface RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) - - # add_executable(adios2_simulations_gray-scott_render-isosurface plot/render_isosurface.cpp) - # target_link_libraries(adios2_simulations_gray-scott_render-isosurface adios2::cxx11_mpi MPI::MPI_C ${VTK_LIBRARIES}) - endif() endif() diff --git a/examples/simulations/gray-scott/analysis/curvature.cpp b/examples/simulations/gray-scott/analysis/curvature.cpp deleted file mode 100644 index 586e2fe7d2..0000000000 --- a/examples/simulations/gray-scott/analysis/curvature.cpp +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * Analysis code for the Gray-Scott simulation. - * Computes mean curvature at each point on an isosurface. - * - * Keichi Takahashi - * - */ - -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../common/timer.hpp" - -vtkSmartPointer read_mesh(const std::vector &bufPoints, - const std::vector &bufCells, - const std::vector &bufNormals) -{ - int nPoints = static_cast(bufPoints.size() / 3); - int nCells = static_cast(bufCells.size() / 3); - - auto points = vtkSmartPointer::New(); - points->SetNumberOfPoints(nPoints); - for (vtkIdType i = 0; i < nPoints; i++) - { - points->SetPoint(i, &bufPoints[i * 3]); - } - - auto polys = vtkSmartPointer::New(); - for (vtkIdType i = 0; i < nCells; i++) - { - vtkIdType a = bufCells[i * 3 + 0]; - vtkIdType b = bufCells[i * 3 + 1]; - vtkIdType c = bufCells[i * 3 + 2]; - - polys->InsertNextCell(3); - polys->InsertCellPoint(a); - polys->InsertCellPoint(b); - polys->InsertCellPoint(c); - } - - auto normals = vtkSmartPointer::New(); - normals->SetNumberOfComponents(3); - for (vtkIdType i = 0; i < nPoints; i++) - { - normals->InsertNextTuple(&bufNormals[i * 3]); - } - - auto polyData = vtkSmartPointer::New(); - polyData->SetPoints(points); - polyData->SetPolys(polys); - polyData->GetPointData()->SetNormals(normals); - - return polyData; -} - -void compute_curvature(const vtkSmartPointer polyData) -{ - vtkSmartPointer curvaturesFilter = vtkSmartPointer::New(); - curvaturesFilter->SetInputData(polyData); - // curvaturesFilter->SetCurvatureTypeToMinimum(); - // curvaturesFilter->SetCurvatureTypeToMaximum(); - // curvaturesFilter->SetCurvatureTypeToGaussian(); - curvaturesFilter->SetCurvatureTypeToMean(); - curvaturesFilter->Update(); -} - -int main(int argc, char *argv[]) -{ - int provided; - MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); - - int rank, procs, wrank; - - MPI_Comm_rank(MPI_COMM_WORLD, &wrank); - - const unsigned int color = 7; - MPI_Comm comm; - MPI_Comm_split(MPI_COMM_WORLD, color, wrank, &comm); - - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &procs); - - if (argc < 3) - { - if (rank == 0) - { - std::cerr << "Too few arguments" << std::endl; - std::cout << "Usage: curvature input output" << std::endl; - } - MPI_Abort(MPI_COMM_WORLD, -1); - } - - const std::string input_fname(argv[1]); - const std::string output_fname(argv[2]); - - adios2::ADIOS adios("adios2.xml", comm); - - adios2::IO inIO = adios.DeclareIO("IsosurfaceOutput"); - adios2::Engine reader = inIO.Open(input_fname, adios2::Mode::Read); - - adios2::IO outIO = adios.DeclareIO("CurvatureOutput"); - adios2::Engine writer = outIO.Open(output_fname, adios2::Mode::Write); - - std::vector points; - std::vector cells; - std::vector normals; - int step; - -#ifdef ENABLE_TIMERS - Timer timer_total; - Timer timer_read; - Timer timer_compute; - - std::ostringstream log_fname; - log_fname << "compute_curvature_pe_" << rank << ".log"; - - std::ofstream log(log_fname.str()); - log << "step\ttotal_curv\tread_curv\tcompute_curv" << std::endl; -#endif - - while (true) - { -#ifdef ENABLE_TIMERS - MPI_Barrier(comm); - timer_total.start(); - timer_read.start(); -#endif - - adios2::StepStatus status = reader.BeginStep(); - - if (status != adios2::StepStatus::OK) - { - break; - } - - auto varPoint = inIO.InquireVariable("point"); - auto varCell = inIO.InquireVariable("cell"); - auto varNormal = inIO.InquireVariable("normal"); - auto varStep = inIO.InquireVariable("step"); - - if (varPoint.Shape().size() > 0 || varCell.Shape().size() > 0) - { - varPoint.SetSelection({{0, 0}, {varPoint.Shape()[0], varPoint.Shape()[1]}}); - varCell.SetSelection({{0, 0}, {varCell.Shape()[0], varCell.Shape()[1]}}); - varNormal.SetSelection({{0, 0}, {varNormal.Shape()[0], varNormal.Shape()[1]}}); - - reader.Get(varPoint, points); - reader.Get(varCell, cells); - reader.Get(varNormal, normals); - } - - reader.Get(varStep, &step); - - reader.EndStep(); - -#ifdef ENABLE_TIMERS - double time_read = timer_read.stop(); - MPI_Barrier(comm); - timer_compute.start(); -#endif - - vtkSmartPointer polyData = read_mesh(points, cells, normals); - compute_curvature(polyData); - - if (!rank) - { - std::cout << "compute_curvature at step " << step << std::endl; - } - -#ifdef ENABLE_TIMERS - double time_compute = timer_compute.stop(); - double time_step = timer_total.stop(); - MPI_Barrier(comm); - - log << step << "\t" << time_step << "\t" << time_read << "\t" << time_compute << std::endl; -#endif - } - -#ifdef ENABLE_TIMERS - log << "total\t" << timer_total.elapsed() << "\t" << timer_read.elapsed() << "\t" - << timer_compute.elapsed() << "\t" << std::endl; - - log.close(); -#endif - - writer.Close(); - reader.Close(); -} diff --git a/examples/simulations/gray-scott/analysis/find_blobs.cpp b/examples/simulations/gray-scott/analysis/find_blobs.cpp deleted file mode 100644 index 8b5c8c54b2..0000000000 --- a/examples/simulations/gray-scott/analysis/find_blobs.cpp +++ /dev/null @@ -1,245 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * Analysis code for the Gray-Scott simulation. - * Reads iso-surface mesh data and detects connected components. - * Counts the total number of connected components and measures the surface - * area of each component. - * - * Keichi Takahashi - * - */ - -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../common/timer.hpp" - -vtkSmartPointer read_mesh(const std::vector &bufPoints, - const std::vector &bufCells, - const std::vector &bufNormals) -{ - int nPoints = static_cast(bufPoints.size() / 3); - int nCells = static_cast(bufCells.size() / 3); - - auto points = vtkSmartPointer::New(); - points->SetNumberOfPoints(nPoints); - for (vtkIdType i = 0; i < nPoints; i++) - { - points->SetPoint(i, &bufPoints[i * 3]); - } - - auto polys = vtkSmartPointer::New(); - for (vtkIdType i = 0; i < nCells; i++) - { - vtkIdType a = bufCells[i * 3 + 0]; - vtkIdType b = bufCells[i * 3 + 1]; - vtkIdType c = bufCells[i * 3 + 2]; - - polys->InsertNextCell(3); - polys->InsertCellPoint(a); - polys->InsertCellPoint(b); - polys->InsertCellPoint(c); - } - - auto normals = vtkSmartPointer::New(); - normals->SetNumberOfComponents(3); - for (vtkIdType i = 0; i < nPoints; i++) - { - normals->InsertNextTuple(&bufNormals[i * 3]); - } - - auto polyData = vtkSmartPointer::New(); - polyData->SetPoints(points); - polyData->SetPolys(polys); - polyData->GetPointData()->SetNormals(normals); - - return polyData; -} - -void find_blobs(const vtkSmartPointer polyData) -{ - auto connectivityFilter = vtkSmartPointer::New(); - connectivityFilter->SetInputData(polyData); - connectivityFilter->SetExtractionModeToAllRegions(); - connectivityFilter->ColorRegionsOn(); - connectivityFilter->Update(); - - int nBlobs = connectivityFilter->GetNumberOfExtractedRegions(); - - std::cout << "Found " << nBlobs << " blobs" << std::endl; - - auto threshold = vtkSmartPointer::New(); - auto massProperties = vtkSmartPointer::New(); - auto surfaceFilter = vtkSmartPointer::New(); - - threshold->SetInputConnection(connectivityFilter->GetOutputPort()); - surfaceFilter->SetInputConnection(threshold->GetOutputPort()); - massProperties->SetInputConnection(surfaceFilter->GetOutputPort()); - - for (int i = 0; i < nBlobs; i++) - { - threshold->SetThresholdFunction(vtkThreshold::THRESHOLD_BETWEEN); - threshold->SetLowerThreshold(i); - threshold->SetUpperThreshold(i); - - std::cout << "Surface area of blob #" << i << " is " << massProperties->GetSurfaceArea() - << std::endl; - } -} - -void find_largest_blob(const vtkSmartPointer polyData) -{ - auto connectivityFilter = vtkSmartPointer::New(); - connectivityFilter->SetInputData(polyData); - connectivityFilter->SetExtractionModeToLargestRegion(); - connectivityFilter->Update(); - - auto massProperties = vtkSmartPointer::New(); - auto surfaceFilter = vtkSmartPointer::New(); - - surfaceFilter->SetInputConnection(connectivityFilter->GetOutputPort()); - massProperties->SetInputConnection(surfaceFilter->GetOutputPort()); - - std::cout << "Surface area of largest blob is " << massProperties->GetSurfaceArea() - << std::endl; -} - -int main(int argc, char *argv[]) -{ - int provided; - MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); - - int rank, procs, wrank; - - MPI_Comm_rank(MPI_COMM_WORLD, &wrank); - - const unsigned int color = 6; - MPI_Comm comm; - MPI_Comm_split(MPI_COMM_WORLD, color, wrank, &comm); - - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &procs); - - if (argc < 2) - { - if (rank == 0) - { - std::cerr << "Too few arguments" << std::endl; - std::cout << "Usage: find_blobs input" << std::endl; - } - MPI_Abort(MPI_COMM_WORLD, -1); - } - - if (procs != 1) - { - if (rank == 0) - { - std::cerr << "find_blobs only supports serial execution" << std::endl; - } - MPI_Abort(MPI_COMM_WORLD, -1); - } - - const std::string input_fname(argv[1]); - - adios2::ADIOS adios("adios2.xml", comm); - - adios2::IO inIO = adios.DeclareIO("IsosurfaceOutput"); - adios2::Engine reader = inIO.Open(input_fname, adios2::Mode::Read); - - std::vector points; - std::vector cells; - std::vector normals; - int step; - -#ifdef ENABLE_TIMERS - Timer timer_total; - Timer timer_compute; - Timer timer_read; - - std::ostringstream log_fname; - log_fname << "find_blobs_pe_" << rank << ".log"; - - std::ofstream log(log_fname.str()); - log << "step\ttotal_blobs\tread_blobs\tcompute_blobs" << std::endl; -#endif - - while (true) - { -#ifdef ENABLE_TIMERS - MPI_Barrier(comm); - timer_total.start(); - timer_read.start(); -#endif - - adios2::StepStatus status = reader.BeginStep(); - - if (status != adios2::StepStatus::OK) - { - break; - } - - auto varPoint = inIO.InquireVariable("point"); - auto varCell = inIO.InquireVariable("cell"); - auto varNormal = inIO.InquireVariable("normal"); - auto varStep = inIO.InquireVariable("step"); - - if (varPoint.Shape().size() > 0 || varCell.Shape().size() > 0) - { - varPoint.SetSelection({{0, 0}, {varPoint.Shape()[0], varPoint.Shape()[1]}}); - varCell.SetSelection({{0, 0}, {varCell.Shape()[0], varCell.Shape()[1]}}); - varNormal.SetSelection({{0, 0}, {varNormal.Shape()[0], varNormal.Shape()[1]}}); - - reader.Get(varPoint, points); - reader.Get(varCell, cells); - reader.Get(varNormal, normals); - } - - reader.Get(varStep, &step); - - reader.EndStep(); - -#ifdef ENABLE_TIMERS - double time_read = timer_read.stop(); - MPI_Barrier(comm); - timer_compute.start(); -#endif - - std::cout << "find_blobs at step " << step << std::endl; - - auto polyData = read_mesh(points, cells, normals); - // find_blobs(polyData); - find_largest_blob(polyData); - -#ifdef ENABLE_TIMERS - double time_compute = timer_compute.stop(); - double time_step = timer_total.stop(); - MPI_Barrier(comm); - - log << step << "\t" << time_step << "\t" << time_read << "\t" << time_compute << std::endl; -#endif - } - -#ifdef ENABLE_TIMERS - log << "total\t" << timer_total.elapsed() << "\t" << timer_read.elapsed() << "\t" - << timer_compute.elapsed() << std::endl; - - log.close(); -#endif - - reader.Close(); -} diff --git a/examples/simulations/gray-scott/analysis/isosurface.cpp b/examples/simulations/gray-scott/analysis/isosurface.cpp deleted file mode 100644 index e02a507aaa..0000000000 --- a/examples/simulations/gray-scott/analysis/isosurface.cpp +++ /dev/null @@ -1,346 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * Analysis code for the Gray-Scott simulation. - * Reads variable U and and extracts the iso-surface using VTK. - * Writes the extracted iso-surface using ADIOS. - * - * Keichi Takahashi - * - */ - -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../common/timer.hpp" - -vtkSmartPointer compute_isosurface(const adios2::Variable &varField, - const std::vector &field, double isovalue) -{ - // Convert field values to vtkImageData - auto importer = vtkSmartPointer::New(); - importer->SetDataSpacing(1, 1, 1); - importer->SetDataOrigin(static_cast(varField.Start()[2]), - static_cast(varField.Start()[1]), - static_cast(varField.Start()[0])); - importer->SetWholeExtent(0, static_cast(varField.Count()[2] - 1), 0, - static_cast(varField.Count()[1] - 1), 0, - static_cast(varField.Count()[0] - 1)); - importer->SetDataExtentToWholeExtent(); - importer->SetDataScalarTypeToDouble(); - importer->SetNumberOfScalarComponents(1); - importer->SetImportVoidPointer(const_cast(field.data())); - - // Run the marching cubes algorithm - auto mcubes = vtkSmartPointer::New(); - mcubes->SetInputConnection(importer->GetOutputPort()); - mcubes->ComputeNormalsOn(); - mcubes->SetValue(0, isovalue); - mcubes->Update(); - - // Return the isosurface as vtkPolyData - return mcubes->GetOutput(); -} - -void write_vtk(const std::string &fname, const vtkSmartPointer polyData) -{ - auto writer = vtkSmartPointer::New(); - writer->SetFileName(fname.c_str()); - writer->SetInputData(polyData); - writer->Write(); -} - -void write_adios(adios2::Engine &writer, const vtkSmartPointer polyData, - adios2::Variable &varPoint, adios2::Variable &varCell, - adios2::Variable &varNormal, adios2::Variable &varOutStep, int step, - MPI_Comm comm) -{ - int numCells = static_cast(polyData->GetNumberOfPolys()); - int numPoints = static_cast(polyData->GetNumberOfPoints()); - int rank; - - MPI_Comm_rank(comm, &rank); - - std::vector points(static_cast(numPoints * 3)); - std::vector normals(static_cast(numPoints * 3)); - std::vector cells(static_cast(numCells * 3)); // Assumes that cells are triangles - - double coords[3]; - - auto cellArray = polyData->GetPolys(); - - cellArray->InitTraversal(); - - // Iterate through cells - for (vtkIdType i = 0; i < polyData->GetNumberOfPolys(); i++) - { - auto idList = vtkSmartPointer::New(); - - cellArray->GetNextCell(idList); - - // Iterate through points of a cell - for (vtkIdType j = 0; j < idList->GetNumberOfIds(); j++) - { - auto id = idList->GetId(j); - - cells[i * 3 + j] = static_cast(id); - - polyData->GetPoint(id, coords); - - points[id * 3 + 0] = coords[0]; - points[id * 3 + 1] = coords[1]; - points[id * 3 + 2] = coords[2]; - } - } - - auto normalArray = polyData->GetPointData()->GetNormals(); - - // Extract normals - for (int i = 0; i < normalArray->GetNumberOfTuples(); i++) - { - normalArray->GetTuple(i, coords); - - normals[i * 3 + 0] = coords[0]; - normals[i * 3 + 1] = coords[1]; - normals[i * 3 + 2] = coords[2]; - } - - int totalPoints, offsetPoints; - MPI_Allreduce(&numPoints, &totalPoints, 1, MPI_INT, MPI_SUM, comm); - MPI_Scan(&numPoints, &offsetPoints, 1, MPI_INT, MPI_SUM, comm); - - writer.BeginStep(); - - varPoint.SetShape( - {static_cast(totalPoints), static_cast(totalPoints > 0 ? 3 : 0)}); - varPoint.SetSelection( - {{static_cast(offsetPoints - numPoints), 0}, - {static_cast(numPoints), static_cast(numPoints > 0 ? 3 : 0)}}); - - varNormal.SetShape(varPoint.Shape()); - varNormal.SetSelection({varPoint.Start(), varPoint.Count()}); - - if (numPoints) - { - writer.Put(varPoint, points.data()); - writer.Put(varNormal, normals.data()); - } - - int totalCells, offsetCells; - MPI_Allreduce(&numCells, &totalCells, 1, MPI_INT, MPI_SUM, comm); - MPI_Scan(&numCells, &offsetCells, 1, MPI_INT, MPI_SUM, comm); - - for (int &cell : cells) - { - cell += (offsetPoints - numPoints); - } - - varCell.SetShape( - {static_cast(totalCells), static_cast(totalCells > 0 ? 3 : 0)}); - varCell.SetSelection( - {{static_cast(offsetCells - numCells), 0}, - {static_cast(numCells), static_cast(numCells > 0 ? 3 : 0)}}); - - if (numCells) - { - writer.Put(varCell, cells.data()); - } - - if (!rank) - { - std::cout << "isosurface at step " << step << " writing out " << totalCells << " cells and " - << totalPoints << " points" << std::endl; - } - - writer.Put(varOutStep, step); - - writer.EndStep(); -} - -int main(int argc, char *argv[]) -{ - int provided; - MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); - - int rank, procs, wrank; - - MPI_Comm_rank(MPI_COMM_WORLD, &wrank); - - const unsigned int color = 5; - MPI_Comm comm; - MPI_Comm_split(MPI_COMM_WORLD, color, wrank, &comm); - - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &procs); - - int dims[3] = {0}; - MPI_Dims_create(procs, 3, dims); - size_t npx = dims[0]; - size_t npy = dims[1]; - size_t npz = dims[2]; - - int coords[3] = {0}; - int periods[3] = {0}; - MPI_Comm cart_comm; - MPI_Cart_create(comm, 3, dims, periods, 0, &cart_comm); - MPI_Cart_coords(cart_comm, rank, 3, coords); - size_t px = coords[0]; - size_t py = coords[1]; - size_t pz = coords[2]; - - if (argc < 4) - { - if (rank == 0) - { - std::cerr << "Too few arguments" << std::endl; - std::cout << "Usage: isosurface input output isovalues..." << std::endl; - } - MPI_Abort(MPI_COMM_WORLD, -1); - } - - const std::string input_fname(argv[1]); - const std::string output_fname(argv[2]); - - std::vector isovalues; - for (int i = 3; i < argc; i++) - { - isovalues.push_back(std::stod(argv[i])); - } - - adios2::ADIOS adios("adios2.xml", comm); - - adios2::IO inIO = adios.DeclareIO("SimulationOutput"); - adios2::Engine reader = inIO.Open(input_fname, adios2::Mode::Read); - - adios2::IO outIO = adios.DeclareIO("IsosurfaceOutput"); - adios2::Engine writer = outIO.Open(output_fname, adios2::Mode::Write); - - auto varPoint = outIO.DefineVariable("point", {1, 3}, {0, 0}, {1, 3}); - auto varCell = outIO.DefineVariable("cell", {1, 3}, {0, 0}, {1, 3}); - auto varNormal = outIO.DefineVariable("normal", {1, 3}, {0, 0}, {1, 3}); - auto varOutStep = outIO.DefineVariable("step"); - - std::vector u; - int step; - -#ifdef ENABLE_TIMERS - Timer timer_total; - Timer timer_read; - Timer timer_compute; - Timer timer_write; - - std::ostringstream log_fname; - log_fname << "isosurface_pe_" << rank << ".log"; - - std::ofstream log(log_fname.str()); - log << "step\ttotal_iso\tread_iso\tcompute_iso\twrite_iso" << std::endl; -#endif - - while (true) - { -#ifdef ENABLE_TIMERS - MPI_Barrier(comm); - timer_total.start(); - timer_read.start(); -#endif - - adios2::StepStatus status = reader.BeginStep(); - - if (status != adios2::StepStatus::OK) - { - break; - } - - adios2::Variable varU = inIO.InquireVariable("U"); - const adios2::Variable varStep = inIO.InquireVariable("step"); - - adios2::Dims shape = varU.Shape(); - - size_t size_x = (shape[0] + npx - 1) / npx; - size_t size_y = (shape[1] + npy - 1) / npy; - size_t size_z = (shape[2] + npz - 1) / npz; - - size_t offset_x = size_x * px; - size_t offset_y = size_y * py; - size_t offset_z = size_z * pz; - - if (px == npx - 1) - { - size_x -= size_x * npx - shape[0]; - } - if (py == npy - 1) - { - size_y -= size_y * npy - shape[1]; - } - if (pz == npz - 1) - { - size_z -= size_z * npz - shape[2]; - } - - varU.SetSelection({{offset_x, offset_y, offset_z}, - {size_x + (px != npx - 1 ? 1 : 0), size_y + (py != npy - 1 ? 1 : 0), - size_z + (pz != npz - 1 ? 1 : 0)}}); - - reader.Get(varU, u); - reader.Get(varStep, step); - reader.EndStep(); - -#ifdef ENABLE_TIMERS - double time_read = timer_read.stop(); - MPI_Barrier(comm); - timer_compute.start(); -#endif - - auto appendFilter = vtkSmartPointer::New(); - - for (const auto isovalue : isovalues) - { - auto polyData = compute_isosurface(varU, u, isovalue); - appendFilter->AddInputData(polyData); - } - - appendFilter->Update(); - -#ifdef ENABLE_TIMERS - double time_compute = timer_compute.stop(); - MPI_Barrier(comm); - timer_write.start(); -#endif - - write_adios(writer, appendFilter->GetOutput(), varPoint, varCell, varNormal, varOutStep, - step, comm); - -#ifdef ENABLE_TIMERS - double time_write = timer_write.stop(); - double time_step = timer_total.stop(); - MPI_Barrier(comm); - - log << step << "\t" << time_step << "\t" << time_read << "\t" << time_compute << "\t" - << time_write << std::endl; -#endif - } - -#ifdef ENABLE_TIMERS - log << "total\t" << timer_total.elapsed() << "\t" << timer_read.elapsed() << "\t" - << timer_compute.elapsed() << "\t" << timer_write.elapsed() << std::endl; - - log.close(); -#endif - - writer.Close(); - reader.Close(); - - MPI_Finalize(); -} diff --git a/examples/simulations/gray-scott/plot/render_isosurface.cpp b/examples/simulations/gray-scott/plot/render_isosurface.cpp deleted file mode 100644 index 30a464b831..0000000000 --- a/examples/simulations/gray-scott/plot/render_isosurface.cpp +++ /dev/null @@ -1,204 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * Visualization code for the Gray-Scott simulation. - * Reads and renders iso-surface mesh data. - * - * Keichi Takahashi - * - */ - -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -VTK_MODULE_INIT(vtkRenderingOpenGL2); - -typedef struct -{ - vtkRenderView *renderView; - vtkPolyDataMapper *mapper; - adios2::IO *inIO; - adios2::Engine *reader; -} Context; - -vtkSmartPointer read_mesh(const std::vector &bufPoints, - const std::vector &bufCells, - const std::vector &bufNormals) -{ - int nPoints = bufPoints.size() / 3; - int nCells = bufCells.size() / 3; - - auto points = vtkSmartPointer::New(); - points->SetNumberOfPoints(nPoints); - for (vtkIdType i = 0; i < nPoints; i++) - { - points->SetPoint(i, &bufPoints[i * 3]); - } - - auto polys = vtkSmartPointer::New(); - for (vtkIdType i = 0; i < nCells; i++) - { - vtkIdType a = bufCells[i * 3 + 0]; - vtkIdType b = bufCells[i * 3 + 1]; - vtkIdType c = bufCells[i * 3 + 2]; - - polys->InsertNextCell(3); - polys->InsertCellPoint(a); - polys->InsertCellPoint(b); - polys->InsertCellPoint(c); - } - - auto normals = vtkSmartPointer::New(); - normals->SetNumberOfComponents(3); - for (vtkIdType i = 0; i < nPoints; i++) - { - normals->InsertNextTuple(&bufNormals[i * 3]); - } - - auto polyData = vtkSmartPointer::New(); - polyData->SetPoints(points); - polyData->SetPolys(polys); - polyData->GetPointData()->SetNormals(normals); - - return polyData; -} - -void timer_func(vtkObject *object, unsigned long eid, void *clientdata, void *calldata) -{ - Context *context = static_cast(clientdata); - - std::vector points; - std::vector cells; - std::vector normals; - int step; - - adios2::StepStatus status = context->reader->BeginStep(); - - if (status != adios2::StepStatus::OK) - { - return; - } - - auto varPoint = context->inIO->InquireVariable("point"); - auto varCell = context->inIO->InquireVariable("cell"); - auto varNormal = context->inIO->InquireVariable("normal"); - auto varStep = context->inIO->InquireVariable("step"); - - if (varPoint.Shape().size() > 0 || varCell.Shape().size() > 0) - { - varPoint.SetSelection({{0, 0}, {varPoint.Shape()[0], varPoint.Shape()[1]}}); - varCell.SetSelection({{0, 0}, {varCell.Shape()[0], varCell.Shape()[1]}}); - varNormal.SetSelection({{0, 0}, {varNormal.Shape()[0], varNormal.Shape()[1]}}); - - context->reader->Get(varPoint, points); - context->reader->Get(varCell, cells); - context->reader->Get(varNormal, normals); - } - - context->reader->Get(varStep, &step); - - context->reader->EndStep(); - - std::cout << "render_isosurface at step " << step << std::endl; - - vtkSmartPointer polyData = read_mesh(points, cells, normals); - - context->mapper->SetInputData(polyData); - context->renderView->ResetCamera(); - context->renderView->Render(); -} - -int main(int argc, char *argv[]) -{ - MPI_Init(&argc, &argv); - - int rank, procs, wrank; - - MPI_Comm_rank(MPI_COMM_WORLD, &wrank); - - const unsigned int color = 7; - MPI_Comm comm; - MPI_Comm_split(MPI_COMM_WORLD, color, wrank, &comm); - - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &procs); - - if (argc < 2) - { - if (rank == 0) - { - std::cerr << "Too few arguments" << std::endl; - std::cout << "Usage: render_isosurface input" << std::endl; - } - MPI_Abort(MPI_COMM_WORLD, -1); - } - - if (procs != 1) - { - if (rank == 0) - { - std::cerr << "render_isosurface only supports serial execution" << std::endl; - } - MPI_Abort(MPI_COMM_WORLD, -1); - } - - const std::string input_fname(argv[1]); - - adios2::ADIOS adios("adios2.xml", comm); - - adios2::IO inIO = adios.DeclareIO("IsosurfaceOutput"); - adios2::Engine reader = inIO.Open(input_fname, adios2::Mode::Read); - - auto mapper = vtkSmartPointer::New(); - - auto actor = vtkSmartPointer::New(); - actor->SetMapper(mapper); - - auto renderView = vtkSmartPointer::New(); - renderView->GetRenderer()->AddActor(actor); - renderView->Update(); - - auto style = vtkSmartPointer::New(); - style->SetCurrentStyleToTrackballCamera(); - - auto interactor = renderView->GetInteractor(); - interactor->Initialize(); - interactor->SetInteractorStyle(style); - interactor->CreateRepeatingTimer(100); - - Context context = { - .renderView = renderView, - .mapper = mapper, - .inIO = &inIO, - .reader = &reader, - }; - - auto timerCallback = vtkSmartPointer::New(); - timerCallback->SetCallback(timer_func); - timerCallback->SetClientData(&context); - interactor->AddObserver(vtkCommand::TimerEvent, timerCallback); - - renderView->Render(); - interactor->Start(); - - reader.Close(); -} From 8d540276e2fcb59086080179de1bb7cf2fbf46f6 Mon Sep 17 00:00:00 2001 From: Spiros Tsalikis Date: Thu, 23 Nov 2023 12:09:26 -0500 Subject: [PATCH 07/18] Enable Shell-Check for lammps contract scripts --- .shellcheck_exclude_paths | 5 ----- testing/contract/lammps/build.sh | 8 ++++++-- testing/contract/lammps/config.sh | 16 +++++++++++----- testing/contract/lammps/install.sh | 8 ++++++-- testing/contract/lammps/setup.sh | 15 +++++++++++---- testing/contract/lammps/test.sh | 13 +++++++++---- 6 files changed, 43 insertions(+), 22 deletions(-) diff --git a/.shellcheck_exclude_paths b/.shellcheck_exclude_paths index e84d523737..6bf6be959f 100644 --- a/.shellcheck_exclude_paths +++ b/.shellcheck_exclude_paths @@ -15,11 +15,6 @@ scripts/developer/setup.sh scripts/docker/setup-user.sh scripts/runconf/runconf.sh scripts/runconf/runconf_olcf.sh -testing/contract/lammps/build.sh -testing/contract/lammps/config.sh -testing/contract/lammps/install.sh -testing/contract/lammps/setup.sh -testing/contract/lammps/test.sh testing/contract/scorpio/build.sh testing/contract/scorpio/config.sh testing/contract/scorpio/install.sh diff --git a/testing/contract/lammps/build.sh b/testing/contract/lammps/build.sh index 0290ec8f3d..436861e621 100755 --- a/testing/contract/lammps/build.sh +++ b/testing/contract/lammps/build.sh @@ -3,6 +3,10 @@ set -x set -e -source $(dirname $(readlink -f ${BASH_SOURCE}))/setup.sh +# shellcheck disable=SC1091 +source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/setup.sh" -cmake --build ${build_dir} -j8 +# Fail if is not set +build_dir="${build_dir:?}" + +cmake --build "${build_dir}" -j8 diff --git a/testing/contract/lammps/config.sh b/testing/contract/lammps/config.sh index 14837d843a..256306608d 100755 --- a/testing/contract/lammps/config.sh +++ b/testing/contract/lammps/config.sh @@ -3,17 +3,23 @@ set -x set -e -source $(dirname $(readlink -f ${BASH_SOURCE}))/setup.sh +# shellcheck disable=SC1091 +source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/setup.sh" -mkdir -p ${build_dir} -cd ${build_dir} +# Fail if is not set +source_dir="${source_dir:?}" +build_dir="${build_dir:?}" +install_dir="${install_dir:?}" + +mkdir -p "${build_dir}" +cd "${build_dir}" cmake \ - -DCMAKE_INSTALL_PREFIX=${install_dir} \ + -DCMAKE_INSTALL_PREFIX="${install_dir}" \ -DBUILD_MPI=yes \ -DBUILD_EXE=yes \ -DBUILD_LIB=no \ -DBUILD_DOC=no \ -DLAMMPS_SIZES=smallbig \ -DPKG_ADIOS=yes \ - ${source_dir}/cmake + "${source_dir}/cmake" diff --git a/testing/contract/lammps/install.sh b/testing/contract/lammps/install.sh index f319d25356..26b65c1592 100755 --- a/testing/contract/lammps/install.sh +++ b/testing/contract/lammps/install.sh @@ -3,6 +3,10 @@ set -x set -e -source $(dirname $(readlink -f ${BASH_SOURCE}))/setup.sh +# shellcheck disable=SC1091 +source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/setup.sh" -cmake --install ${build_dir} +# Fail if is not set +build_dir="${build_dir:?}" + +cmake --install "${build_dir}" diff --git a/testing/contract/lammps/setup.sh b/testing/contract/lammps/setup.sh index ca74ccf68a..00a4a7f1d5 100755 --- a/testing/contract/lammps/setup.sh +++ b/testing/contract/lammps/setup.sh @@ -1,7 +1,14 @@ -source_dir=$(readlink -f ${PWD})/source -build_dir=$(readlink -f ${PWD})/build -install_dir=$(readlink -f ${PWD})/install -test_dir=$(readlink -f ${PWD})/test +#!/bin/bash + +source_dir=$(readlink -f "${PWD}")/source +build_dir=$(readlink -f "${PWD}")/build +install_dir=$(readlink -f "${PWD}")/install +test_dir=$(readlink -f "${PWD}")/test + +export source_dir +export build_dir +export install_dir +export test_dir echo "source_dir = \"${source_dir}\"" echo "build_dir = \"${build_dir}\"" diff --git a/testing/contract/lammps/test.sh b/testing/contract/lammps/test.sh index 2c378c1fda..28a0486b34 100755 --- a/testing/contract/lammps/test.sh +++ b/testing/contract/lammps/test.sh @@ -3,13 +3,18 @@ set -x set -e -source $(dirname $(readlink -f ${BASH_SOURCE}))/setup.sh +# shellcheck disable=SC1091 +source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/setup.sh" -mkdir -p ${test_dir} -cd ${test_dir} +# Fail if is not set +install_dir="${install_dir:?}" +test_dir="${test_dir:?}" + +mkdir -p "${test_dir}" +cd "${test_dir}" cp -v /opt/adios2/source/testing/contract/lammps/{adios2_config.xml,check_results.sh,in.test} . -mpiexec -np 4 --oversubscribe ${install_dir}/bin/lmp -in in.test +mpiexec -np 4 --oversubscribe "${install_dir}/bin/lmp" -in in.test ./check_results.sh From af1d3d230fd4fceed350799af917ec12b670ebcf Mon Sep 17 00:00:00 2001 From: Spiros Tsalikis Date: Thu, 23 Nov 2023 12:10:06 -0500 Subject: [PATCH 08/18] Enable Shell-Check for scorpio contract scripts --- .shellcheck_exclude_paths | 5 ----- testing/contract/scorpio/build.sh | 8 ++++++-- testing/contract/scorpio/config.sh | 18 ++++++++++++------ testing/contract/scorpio/install.sh | 8 ++++++-- testing/contract/scorpio/setup.sh | 15 +++++++++++---- testing/contract/scorpio/test.sh | 13 +++++++++---- 6 files changed, 44 insertions(+), 23 deletions(-) diff --git a/.shellcheck_exclude_paths b/.shellcheck_exclude_paths index 6bf6be959f..bf1b78e8af 100644 --- a/.shellcheck_exclude_paths +++ b/.shellcheck_exclude_paths @@ -15,11 +15,6 @@ scripts/developer/setup.sh scripts/docker/setup-user.sh scripts/runconf/runconf.sh scripts/runconf/runconf_olcf.sh -testing/contract/scorpio/build.sh -testing/contract/scorpio/config.sh -testing/contract/scorpio/install.sh -testing/contract/scorpio/setup.sh -testing/contract/scorpio/test.sh testing/contract/tau/build.sh testing/contract/tau/config.sh testing/contract/tau/install.sh diff --git a/testing/contract/scorpio/build.sh b/testing/contract/scorpio/build.sh index 0290ec8f3d..436861e621 100755 --- a/testing/contract/scorpio/build.sh +++ b/testing/contract/scorpio/build.sh @@ -3,6 +3,10 @@ set -x set -e -source $(dirname $(readlink -f ${BASH_SOURCE}))/setup.sh +# shellcheck disable=SC1091 +source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/setup.sh" -cmake --build ${build_dir} -j8 +# Fail if is not set +build_dir="${build_dir:?}" + +cmake --build "${build_dir}" -j8 diff --git a/testing/contract/scorpio/config.sh b/testing/contract/scorpio/config.sh index bb88cc834c..9ff170bd49 100755 --- a/testing/contract/scorpio/config.sh +++ b/testing/contract/scorpio/config.sh @@ -3,22 +3,28 @@ set -x set -e -source $(dirname $(readlink -f ${BASH_SOURCE}))/setup.sh +# shellcheck disable=SC1091 +source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/setup.sh" -mkdir -p ${build_dir} -cd ${build_dir} +# Fail if is not set +source_dir="${source_dir:?}" +build_dir="${build_dir:?}" +install_dir="${install_dir:?}" + +mkdir -p "${build_dir}" +cd "${build_dir}" export CC=mpicc export CXX=mpic++ export FC=mpifort cmake \ - -DCMAKE_INSTALL_PREFIX=${install_dir} \ + -DCMAKE_INSTALL_PREFIX="${install_dir}" \ -DFPHSA_NAME_MISMATCHED=ON \ -DPIO_ENABLE_TESTS=ON \ -DPIO_ENABLE_EXAMPLES=ON \ -DWITH_NETCDF=OFF \ -DWITH_PNETCDF=ON \ - -DPnetCDF_PATH=$(spack location -i parallel-netcdf) \ + -DPnetCDF_PATH="$(spack location -i parallel-netcdf)" \ -DWITH_ADIOS2=ON \ - ${source_dir} + "${source_dir}" diff --git a/testing/contract/scorpio/install.sh b/testing/contract/scorpio/install.sh index f319d25356..26b65c1592 100755 --- a/testing/contract/scorpio/install.sh +++ b/testing/contract/scorpio/install.sh @@ -3,6 +3,10 @@ set -x set -e -source $(dirname $(readlink -f ${BASH_SOURCE}))/setup.sh +# shellcheck disable=SC1091 +source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/setup.sh" -cmake --install ${build_dir} +# Fail if is not set +build_dir="${build_dir:?}" + +cmake --install "${build_dir}" diff --git a/testing/contract/scorpio/setup.sh b/testing/contract/scorpio/setup.sh index 08fbcaa5e2..1e32491be1 100755 --- a/testing/contract/scorpio/setup.sh +++ b/testing/contract/scorpio/setup.sh @@ -1,7 +1,14 @@ -source_dir=$(readlink -f ${PWD})/source -build_dir=$(readlink -f ${PWD})/build -install_dir=$(readlink -f ${PWD})/install -test_dir=$(readlink -f ${PWD})/test +#!/bin/bash + +source_dir=$(readlink -f "${PWD}")/source +build_dir=$(readlink -f "${PWD}")/build +install_dir=$(readlink -f "${PWD}")/install +test_dir=$(readlink -f "${PWD}")/test + +export source_dir +export build_dir +export install_dir +export test_dir echo "source_dir = \"${source_dir}\"" echo "build_dir = \"${build_dir}\"" diff --git a/testing/contract/scorpio/test.sh b/testing/contract/scorpio/test.sh index 0249fa7d5f..94b4fca85d 100755 --- a/testing/contract/scorpio/test.sh +++ b/testing/contract/scorpio/test.sh @@ -3,12 +3,17 @@ set -x set -e -source $(dirname $(readlink -f ${BASH_SOURCE}))/setup.sh +# shellcheck disable=SC1091 +source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/setup.sh" -mkdir -p ${test_dir} -cd ${test_dir} +# Fail if is not set +build_dir="${build_dir:?}" +test_dir="${test_dir:?}" -mpiexec --oversubscribe -np 4 ${build_dir}/examples/adios/example3 -v +mkdir -p "${test_dir}" +cd "${test_dir}" + +mpiexec --oversubscribe -np 4 "${build_dir}/examples/adios/example3" -v bpls -d example3_1.nc.bp.dir/example3_1.nc.bp.0 > 0.dump diff -u 0.dump /opt/adios2/source/testing/contract/scorpio/0.dump From fab66d4bd4d33de87ce8f73ff0269dcc582339b2 Mon Sep 17 00:00:00 2001 From: Spiros Tsalikis Date: Thu, 23 Nov 2023 12:10:23 -0500 Subject: [PATCH 09/18] Enable Shell-Check for tau contract scripts --- .shellcheck_exclude_paths | 5 ----- testing/contract/tau/build.sh | 8 ++++++-- testing/contract/tau/config.sh | 16 ++++++++++------ testing/contract/tau/install.sh | 8 ++++++-- testing/contract/tau/setup.sh | 13 ++++++++++--- testing/contract/tau/test.sh | 13 +++++++++---- 6 files changed, 41 insertions(+), 22 deletions(-) diff --git a/.shellcheck_exclude_paths b/.shellcheck_exclude_paths index bf1b78e8af..1ef95d56a2 100644 --- a/.shellcheck_exclude_paths +++ b/.shellcheck_exclude_paths @@ -15,8 +15,3 @@ scripts/developer/setup.sh scripts/docker/setup-user.sh scripts/runconf/runconf.sh scripts/runconf/runconf_olcf.sh -testing/contract/tau/build.sh -testing/contract/tau/config.sh -testing/contract/tau/install.sh -testing/contract/tau/setup.sh -testing/contract/tau/test.sh diff --git a/testing/contract/tau/build.sh b/testing/contract/tau/build.sh index 0290ec8f3d..436861e621 100755 --- a/testing/contract/tau/build.sh +++ b/testing/contract/tau/build.sh @@ -3,6 +3,10 @@ set -x set -e -source $(dirname $(readlink -f ${BASH_SOURCE}))/setup.sh +# shellcheck disable=SC1091 +source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/setup.sh" -cmake --build ${build_dir} -j8 +# Fail if is not set +build_dir="${build_dir:?}" + +cmake --build "${build_dir}" -j8 diff --git a/testing/contract/tau/config.sh b/testing/contract/tau/config.sh index a89b2deece..0fd803a1a6 100755 --- a/testing/contract/tau/config.sh +++ b/testing/contract/tau/config.sh @@ -3,11 +3,15 @@ set -x set -e -source $(dirname $(readlink -f ${BASH_SOURCE}))/setup.sh +# shellcheck disable=SC1091 +source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/setup.sh" -mkdir -p ${build_dir} -cd ${build_dir} +# Fail if is not set +source_dir="${source_dir:?}" +build_dir="${build_dir:?}" +install_dir="${install_dir:?}" -cmake \ - -DCMAKE_INSTALL_PREFIX=${install_dir} \ - ${source_dir} +mkdir -p "${build_dir}" +cd "${build_dir}" + +cmake -DCMAKE_INSTALL_PREFIX="${install_dir}" "${source_dir}" diff --git a/testing/contract/tau/install.sh b/testing/contract/tau/install.sh index f319d25356..26b65c1592 100755 --- a/testing/contract/tau/install.sh +++ b/testing/contract/tau/install.sh @@ -3,6 +3,10 @@ set -x set -e -source $(dirname $(readlink -f ${BASH_SOURCE}))/setup.sh +# shellcheck disable=SC1091 +source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/setup.sh" -cmake --install ${build_dir} +# Fail if is not set +build_dir="${build_dir:?}" + +cmake --install "${build_dir}" diff --git a/testing/contract/tau/setup.sh b/testing/contract/tau/setup.sh index d79a4d9ee9..a9e1aff92d 100755 --- a/testing/contract/tau/setup.sh +++ b/testing/contract/tau/setup.sh @@ -1,7 +1,14 @@ +#!/bin/bash + source_dir="/opt/adios2/source/examples/basics/variablesShapes" -build_dir=$(readlink -f ${PWD})/build -install_dir=$(readlink -f ${PWD})/install -test_dir=$(readlink -f ${PWD})/test +build_dir=$(readlink -f "${PWD}")/build +install_dir=$(readlink -f "${PWD}")/install +test_dir=$(readlink -f "${PWD}")/test + +export source_dir +export build_dir +export install_dir +export test_dir echo "source_dir = \"${source_dir}\"" echo "build_dir = \"${build_dir}\"" diff --git a/testing/contract/tau/test.sh b/testing/contract/tau/test.sh index a696145de3..47ac7b1769 100755 --- a/testing/contract/tau/test.sh +++ b/testing/contract/tau/test.sh @@ -3,14 +3,19 @@ set -x set -e -source $(dirname $(readlink -f ${BASH_SOURCE}))/setup.sh +# shellcheck disable=SC1091 +source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/setup.sh" -mkdir -p ${test_dir} -cd ${test_dir} +# Fail if is not set +install_dir="${install_dir:?}" +test_dir="${test_dir:?}" + +mkdir -p "${test_dir}" +cd "${test_dir}" TAU=$(spack location -i tau)/bin/tau_exec -mpiexec -np 2 ${TAU} ${install_dir}/bin/adios2_basics_variablesShapes +mpiexec -np 2 "${TAU}" "${install_dir}/bin/adios2_basics_variablesShapes" [ ! -f profile.0.0.0 ] || [ ! -s profile.0.0.0 ] && { echo "Error: file profile.0.0.0 not found or empty"; exit 1; } [ ! -f profile.1.0.0 ] || [ ! -s profile.1.0.0 ] && { echo "Error: file profile.1.0.0 not found or empty"; exit 1; } From 3518006e7520d9e089ded1bbf2d112af4aeb77e8 Mon Sep 17 00:00:00 2001 From: Spiros Tsalikis Date: Thu, 23 Nov 2023 12:14:19 -0500 Subject: [PATCH 10/18] Enable Shell-Check for circle CI scripts --- .shellcheck_exclude_paths | 2 -- scripts/ci/circle/postCDashStatus.sh | 7 ++++--- scripts/ci/circle/run.sh | 11 +++++++---- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.shellcheck_exclude_paths b/.shellcheck_exclude_paths index 1ef95d56a2..4f77bacc4d 100644 --- a/.shellcheck_exclude_paths +++ b/.shellcheck_exclude_paths @@ -1,5 +1,3 @@ -scripts/ci/circle/postCDashStatus.sh -scripts/ci/circle/run.sh scripts/ci/gh-actions/check-branch-name.sh scripts/ci/gh-actions/get-changed-files.sh scripts/ci/gh-actions/macos-setup.sh diff --git a/scripts/ci/circle/postCDashStatus.sh b/scripts/ci/circle/postCDashStatus.sh index 800b192d77..031ae912cc 100755 --- a/scripts/ci/circle/postCDashStatus.sh +++ b/scripts/ci/circle/postCDashStatus.sh @@ -4,7 +4,7 @@ USER=${STATUS_ROBOT_NAME} TOKEN=${STATUS_ROBOT_KEY} COMMIT=${CIRCLE_SHA1} CDASH_STATUS_CONTEXT="cdash" -SOURCE_DIR="$(readlink -f ${CIRCLE_WORKING_DIRECTORY}/source)" +SOURCE_DIR="$(readlink -f "${CIRCLE_WORKING_DIRECTORY}"/source)" build_status_body() { cat < /dev/null)" - if [ $? -eq 0 ] + RESULT="$(curl -s "${APIURL}" | python3 -c "import sys, json; print(json.load(sys.stdin)['head']['ref'])" 2> /dev/null)" + exit_status=$? + if [ "$exit_status" -eq 0 ] then REALBRANCH="$(echo "${RESULT}" | tr '/' '-')" fi @@ -70,8 +72,9 @@ echo "**********Env End************" echo "**********CTest Begin**********" ${CTEST} --version -echo ${CTEST} -VV -S ${CTEST_SCRIPT} -Ddashboard_full=OFF ${CTEST_STEP_ARGS} -${CTEST} -VV -S ${CTEST_SCRIPT} -Ddashboard_full=OFF ${CTEST_STEP_ARGS} +echo ${CTEST} -VV -S "${CTEST_SCRIPT}" -Ddashboard_full=OFF "${CTEST_STEP_ARGS}" +# shellcheck disable=SC2086 +${CTEST} -VV -S "${CTEST_SCRIPT}" -Ddashboard_full=OFF ${CTEST_STEP_ARGS} RET=$? echo "**********CTest End************" From ea4a249140a9f77eeb334a2233dfe3e3fb6c64b2 Mon Sep 17 00:00:00 2001 From: Spiros Tsalikis Date: Thu, 23 Nov 2023 12:40:36 -0500 Subject: [PATCH 11/18] Enable Shell-Check for gh-actions scripts --- .shellcheck_exclude_paths | 4 ---- scripts/ci/gh-actions/check-branch-name.sh | 2 +- scripts/ci/gh-actions/get-changed-files.sh | 12 ++++++------ scripts/ci/gh-actions/macos-setup.sh | 8 ++++---- scripts/ci/gh-actions/run.sh | 16 +++++++++------- 5 files changed, 20 insertions(+), 22 deletions(-) diff --git a/.shellcheck_exclude_paths b/.shellcheck_exclude_paths index 4f77bacc4d..873e54f928 100644 --- a/.shellcheck_exclude_paths +++ b/.shellcheck_exclude_paths @@ -1,7 +1,3 @@ -scripts/ci/gh-actions/check-branch-name.sh -scripts/ci/gh-actions/get-changed-files.sh -scripts/ci/gh-actions/macos-setup.sh -scripts/ci/gh-actions/run.sh scripts/ci/scripts/github-prs-to-gitlab.sh scripts/ci/scripts/run-clang-format.sh scripts/ci/scripts/run-flake8.sh diff --git a/scripts/ci/gh-actions/check-branch-name.sh b/scripts/ci/gh-actions/check-branch-name.sh index 83bffd8e85..52d09e6cb0 100755 --- a/scripts/ci/gh-actions/check-branch-name.sh +++ b/scripts/ci/gh-actions/check-branch-name.sh @@ -4,7 +4,7 @@ if [ "${GITHUB_EVENT_NAME}" = "pull_request" ] then if [ -z "${BASE_REF}" ] then - BASE_REF="$(jq -r .pull_request.base.ref ${GITHUB_EVENT_PATH})" + BASE_REF="$(jq -r .pull_request.base.ref "${GITHUB_EVENT_PATH}")" fi echo "Base ref: ${BASE_REF}" echo "Head ref: ${GITHUB_HEAD_REF}" diff --git a/scripts/ci/gh-actions/get-changed-files.sh b/scripts/ci/gh-actions/get-changed-files.sh index 9e3ed8fa5b..5afcd19396 100755 --- a/scripts/ci/gh-actions/get-changed-files.sh +++ b/scripts/ci/gh-actions/get-changed-files.sh @@ -3,12 +3,12 @@ case "${GITHUB_EVENT_NAME}" in pull_request) - BASE_SHA=$(jq -r .pull_request.base.sha ${GITHUB_EVENT_PATH}) - HEAD_SHA=$(jq -r .pull_request.head.sha ${GITHUB_EVENT_PATH}) + BASE_SHA=$(jq -r .pull_request.base.sha "${GITHUB_EVENT_PATH}") + HEAD_SHA=$(jq -r .pull_request.head.sha "${GITHUB_EVENT_PATH}") ;; push) - BASE_SHA=$(jq -r .before ${GITHUB_EVENT_PATH}) - HEAD_SHA=$(jq -r .after ${GITHUB_EVENT_PATH}) + BASE_SHA=$(jq -r .before "${GITHUB_EVENT_PATH}") + HEAD_SHA=$(jq -r .after "${GITHUB_EVENT_PATH}") ;; *) echo "Unable to get changed files from '${GITHUB_EVENT_NAME}' event" @@ -20,11 +20,11 @@ echo "Base: ${BASE_SHA}" echo "Head: ${HEAD_SHA}" echo "" -git fetch origin ${BASE_SHA} +git fetch origin "${BASE_SHA}" echo "" echo "::group::All changed files" -git diff --name-only ${BASE_SHA}...${HEAD_SHA} | tee all-changed-files.txt +git diff --name-only "${BASE_SHA}"..."${HEAD_SHA}" | tee all-changed-files.txt echo "::group::Filtered changes" grep -v '^docs/' all-changed-files.txt | tee filtered-changed-files.txt diff --git a/scripts/ci/gh-actions/macos-setup.sh b/scripts/ci/gh-actions/macos-setup.sh index 8cae241e0e..0eb6850f20 100755 --- a/scripts/ci/gh-actions/macos-setup.sh +++ b/scripts/ci/gh-actions/macos-setup.sh @@ -8,13 +8,13 @@ then echo "Error: GH_YML_MATRIX_COMPILER variable is not defined" exit 1 fi -XCODE_VER="$(echo ${GH_YML_MATRIX_COMPILER} | sed -e 's|_|.|g' -e 's|xcode||')" -if [ ! -d /Applications/Xcode_${XCODE_VER}.app ] +XCODE_VER="$(echo "${GH_YML_MATRIX_COMPILER}" | sed -e 's|_|.|g' -e 's|xcode||')" +if [ ! -d "/Applications/Xcode_${XCODE_VER}.app" ] then echo "Error: XCode installation directory /Applications/Xcode_${XCODE_VER}.app does not exist" exit 2 fi -sudo xcode-select --switch /Applications/Xcode_${XCODE_VER}.app +sudo xcode-select --switch "/Applications/Xcode_${XCODE_VER}.app" echo "Installing CMake" @@ -36,7 +36,7 @@ brew install ninja echo "Installing GCC" brew install gcc -sudo ln -v -s $(which gfortran-11) /usr/local/bin/gfortran +sudo ln -v -s "$(which gfortran-11)" /usr/local/bin/gfortran echo "Installing blosc compression" brew install c-blosc diff --git a/scripts/ci/gh-actions/run.sh b/scripts/ci/gh-actions/run.sh index 5dcfcca7aa..c511730eea 100755 --- a/scripts/ci/gh-actions/run.sh +++ b/scripts/ci/gh-actions/run.sh @@ -100,19 +100,19 @@ export ADIOS2_IP=127.0.0.1 export OMP_NUM_THREADS=1 # Load any additional setup scripts -if [ -f gha/scripts/ci/setup-run/ci-${GH_YML_JOBNAME}.sh ] +if [ -f "gha/scripts/ci/setup-run/ci-${GH_YML_JOBNAME}.sh" ] then SETUP_RUN_SCRIPT=gha/scripts/ci/setup-run/ci-${GH_YML_JOBNAME}.sh -elif [ -f gha/scripts/ci/setup-run/ci-${GH_YML_MATRIX_OS}-${GH_YML_MATRIX_COMPILER}-${GH_YML_MATRIX_PARALLEL}.sh ] +elif [ -f "gha/scripts/ci/setup-run/ci-${GH_YML_MATRIX_OS}-${GH_YML_MATRIX_COMPILER}-${GH_YML_MATRIX_PARALLEL}.sh" ] then SETUP_RUN_SCRIPT=gha/scripts/ci/setup-run/ci-${GH_YML_MATRIX_OS}-${GH_YML_MATRIX_COMPILER}-${GH_YML_MATRIX_PARALLEL}.sh -elif [ -f gha/scripts/ci/setup-run/ci-${GH_YML_MATRIX_OS}-${GH_YML_MATRIX_COMPILER}.sh ] +elif [ -f "gha/scripts/ci/setup-run/ci-${GH_YML_MATRIX_OS}-${GH_YML_MATRIX_COMPILER}.sh" ] then SETUP_RUN_SCRIPT=gha/scripts/ci/setup-run/ci-${GH_YML_MATRIX_OS}-${GH_YML_MATRIX_COMPILER}.sh -elif [ -f gha/scripts/ci/setup-run/ci-${GH_YML_MATRIX_OS}.sh ] +elif [ -f "gha/scripts/ci/setup-run/ci-${GH_YML_MATRIX_OS}.sh" ] then SETUP_RUN_SCRIPT=gha/scripts/ci/setup-run/ci-${GH_YML_MATRIX_OS}.sh -elif [ -f gha/scripts/ci/setup-run/ci-${GH_YML_BASE_OS}.sh ] +elif [ -f "gha/scripts/ci/setup-run/ci-${GH_YML_BASE_OS}.sh" ] then SETUP_RUN_SCRIPT=gha/scripts/ci/setup-run/ci-${GH_YML_BASE_OS}.sh fi @@ -126,7 +126,8 @@ echo "::endgroup::" echo "::group::Job-run setup (if any)" if [ "${SETUP_RUN_SCRIPT:-UNSET}" != "UNSET" ] then - source ${SETUP_RUN_SCRIPT} + # shellcheck source=/dev/null + source "${SETUP_RUN_SCRIPT}" fi echo "::endgroup::" @@ -139,7 +140,8 @@ echo "::group::CTest version" echo "::endgroup::" echo "::group::Execute job step" -"${CTEST}" -VV -S ${CTEST_SCRIPT} -Ddashboard_full=OFF ${CTEST_STEP_ARGS} +# shellcheck disable=SC2086 +"${CTEST}" -VV -S "${CTEST_SCRIPT}" -Ddashboard_full=OFF ${CTEST_STEP_ARGS} RET=$? echo "::endgroup::" From 8608588651bfb4ed96a6b7afe072d91c1fad1c40 Mon Sep 17 00:00:00 2001 From: Spiros Tsalikis Date: Thu, 23 Nov 2023 11:05:49 -0500 Subject: [PATCH 12/18] Add Section for Anatomy of an ADIOS Program --- docs/user_guide/source/components/anatomy.rst | 136 ++++++++++++++++++ .../source/components/components.rst | 1 + 2 files changed, 137 insertions(+) create mode 100644 docs/user_guide/source/components/anatomy.rst diff --git a/docs/user_guide/source/components/anatomy.rst b/docs/user_guide/source/components/anatomy.rst new file mode 100644 index 0000000000..0832b6dc3c --- /dev/null +++ b/docs/user_guide/source/components/anatomy.rst @@ -0,0 +1,136 @@ +.. _sec:basics_interface_components_anatomy: + +*************************** +Anatomy of an ADIOS Program +*************************** + +Anatomy of an ADIOS Output +-------------------------- + +.. code:: C++ + + ADIOS adios("config.xml", MPI_COMM_WORLD); + | + | IO io = adios.DeclareIO(...); + | | + | | Variable<...> var = io.DefineVariable<...>(...) + | | Attribute<...> attr = io.DefineAttribute<...>(...) + | | Engine e = io.Open("OutputFileName.bp", adios2::Mode::Write); + | | | + | | | e.BeginStep() + | | | | + | | | | e.Put(var, datapointer); + | | | | + | | | e.EndStep() + | | | + | | e.Close(); + | | + | |--> IO goes out of scope + | + |--> ADIOS goes out of scope or adios2_finalize() + + +The pseudo code above depicts the basic structure of performing output. The ``ADIOS`` object is necessary to hold all +other objects. It is initialized with an MPI communicator in a parallel program or without in a serial program. +Additionally, a config file (XML or YAML format) can be specified here to load runtime configuration. Only one ADIOS +object is needed throughout the entire application but you can create as many as you want (e.g. if you need to separate +IO objects using the same name in a program that reads similar input from an ensemble of multiple applications). + +The ``IO`` object is required to hold the variable and attribute definitions, and runtime options for a particular input +or output stream. The IO object has a name, which is used only to refer to runtime options in the configuration file. +One IO object can only be used in one output or input stream. The only exception where an IO object can be used twice is +one input stream plus one output stream where the output is reusing the variable definitions loaded during input. + +``Variable`` and ``Attribute`` definitions belong to one IO object, which means, they can only be used in one output. +You need to define new ones for other outputs. Just because a Variable is defined, it will not appear in the output +unless an associated Put() call provides the content. + +A stream is opened and closed once. The ``Engine`` object implements the data movement for the stream. It depends on the +runtime options of the IO object that what type of an engine is created in the Open() call. One output step is denoted +by a pair of BeginStep..EndStep block. + +An output step consist of variables and attributes. Variables are just definitions without content, so one must call a +Put() function to provide the application data pointer that contains the data content one wants to write out. Attributes +have their content in their definitions so there is no need for an extra call. + +Some rules: + +* Variables can be defined any time, before the corresponding Put() call +* Attributes can be defined any time before EndStep +* The following functions must be treated as Collective operations + + * ADIOS + * Open + * BeginStep + * EndStep + * Close + +.. note:: + + If there is only one output step, and we only want to write it to a file on disk, never stream it to other + application, then BeginStep and EndStep are not required but it does not make any difference if they are called. + +Anatomy of an ADIOS Input +------------------------- + +.. code:: C++ + + ADIOS adios("config.xml", MPI_COMM_WORLD); + | + | IO io = adios.DeclareIO(...); + | | + | | Engine e = io.Open("InputFileName.bp", adios2::Mode::Read); + | | | + | | | e.BeginStep() + | | | | + | | | | varlist = io.AvailableVariables(...) + | | | | Variable var = io.InquireVariable(...) + | | | | Attribute attr = io.InquireAttribute(...) + | | | | | + | | | | | e.Get(var, datapointer); + | | | | | + | | | | + | | | e.EndStep() + | | | + | | e.Close(); + | | + | |--> IO goes out of scope + | + |--> ADIOS goes out of scope or adios2_finalize() + +The difference between input and output is that while we have to define the variables and attributes for an output, we +have to retrieve the available variables in an input first as definitions (Variable and Attribute objects). + +If we know the particular variable (name and type) in the input stream, we can get the definition using +InquireVariable(). Generic tools that process any input must use other functions to retrieve the list of variable names +and their types first and then get the individual Variable objects. The same is true for Attributes. + +Anatomy of an ADIOS File-only Input +----------------------------------- + +Previously we explored how to read using the input mode `adios2::Mode::Read`. Nonetheless, ADIOS has another input mode +named `adios2::Mode::ReadRandomAccess`. `adios2::Mode::Read` mode allows data access only timestep by timestep using +`BeginStep/EndStep`, but generally it is more memory efficient as ADIOS is only required to load metadata for the +current timestep. `ReadRandomAccess` can only be used with file engines and involves loading all the file metadata at +once. So it can be more memory intensive than `adios2::Mode::Read` mode, but allows reading data from any timestep using +`SetStepSelection()`. If you use `adios2::Mode::ReadRandomAccess` mode, be sure to allocate enough memory to hold +multiple steps of the variable content. + +.. code:: C++ + + ADIOS adios("config.xml", MPI_COMM_WORLD); + | + | IO io = adios.DeclareIO(...); + | | + | | Engine e = io.Open("InputFileName.bp", adios2::Mode::ReadRandomAccess); + | | | + | | | Variable var = io.InquireVariable(...) + | | | | var.SetStepSelection() + | | | | e.Get(var, datapointer); + | | | | + | | | + | | e.Close(); + | | + | |--> IO goes out of scope + | + |--> ADIOS goes out of scope or adios2_finalize() diff --git a/docs/user_guide/source/components/components.rst b/docs/user_guide/source/components/components.rst index 97b4f0b277..791a149a0f 100644 --- a/docs/user_guide/source/components/components.rst +++ b/docs/user_guide/source/components/components.rst @@ -10,3 +10,4 @@ Interface Components .. include:: engine.rst .. include:: operator.rst .. include:: runtime.rst +.. include:: anatomy.rst From bf7b229ab382b2411f5ef7cc62fbf25f9889fade Mon Sep 17 00:00:00 2001 From: Spiros Tsalikis Date: Thu, 23 Nov 2023 11:47:23 -0500 Subject: [PATCH 13/18] Add Global Array Capabilities and Limitations --- .../user_guide/source/components/variable.rst | 96 ++++++++++++++++--- 1 file changed, 85 insertions(+), 11 deletions(-) diff --git a/docs/user_guide/source/components/variable.rst b/docs/user_guide/source/components/variable.rst index baf071defb..69adbc884d 100644 --- a/docs/user_guide/source/components/variable.rst +++ b/docs/user_guide/source/components/variable.rst @@ -6,15 +6,17 @@ An ``adios2::Variable`` is the link between a piece of data coming from an appli This component handles all application variables classified by data type and shape. Each ``IO`` holds a set of Variables, and each ``Variable`` is identified with a unique name. -They are created using the reference from ``IO::DefineVariable`` or retrieved using the pointer from ``IO::InquireVariable`` functions in :ref:`IO`. +They are created using the reference from ``IO::DefineVariable`` or retrieved using the pointer from +``IO::InquireVariable`` functions in :ref:`IO`. Data Types --------------------- +---------- Only primitive types are supported in ADIOS2. -Fixed-width types from ` and `_ should be preferred when writing portable code. -ADIOS2 maps primitive types to equivalent fixed-width types (e.g. ``int`` -> ``int32_t``). -In C++, acceptable types ``T`` in ``Variable`` along with their preferred fix-width equivalent in 64-bit platforms are given below: +Fixed-width types from ` and `_ should be +preferred when writing portable code. ADIOS2 maps primitive types to equivalent fixed-width types +(e.g. ``int`` -> ``int32_t``). In C++, acceptable types ``T`` in ``Variable`` along with their preferred fix-width +equivalent in 64-bit platforms are given below: .. code-block:: c++ @@ -52,19 +54,19 @@ In C++, acceptable types ``T`` in ``Variable`` along with their preferred fix Python APIs: use the equivalent fixed-width types from numpy. If ``dtype`` is not specified, ADIOS2 handles numpy defaults just fine as long as primitive types are passed. - Shapes ---------------------- +------ ADIOS2 is designed for MPI applications. Thus different application data shapes must be supported depending on their scope within a particular MPI communicator. -The shape is defined at creation from the ``IO`` object by providing the dimensions: shape, start, count in the ``IO::DefineVariable``. -The supported shapes are described below. +The shape is defined at creation from the ``IO`` object by providing the dimensions: shape, start, count in the +``IO::DefineVariable``. The supported shapes are described below. 1. **Global Single Value**: Only a name is required for their definition. -These variables are helpful for storing global information, preferably managed by only one MPI process, that may or may not change over steps: *e.g.* total number of particles, collective norm, number of nodes/cells, etc. +These variables are helpful for storing global information, preferably managed by only one MPI process, that may or may +not change over steps: *e.g.* total number of particles, collective norm, number of nodes/cells, etc. .. code-block:: c++ @@ -157,8 +159,80 @@ be applicable to it. JoinedArrays are currently only supported by BP4 and BP5 engines, as well as the SST engine with BP5 marshalling. - +Global Array Capabilities and Limitations +----------------------------------------- + +ADIOS2 is focusing on writing and reading N-dimensional, distributed, global arrays of primitive types. The basic idea +is that, usually, a simulation has such a data structure in memory (distributed across multiple processes) and wants to +dump its content regularly as it progresses. ADIOS2 was designed to: + +1. to do this writing and reading as fast as possible +2. to enable reading any subsection of the array + +.. image:: https://imgur.com/6nX67yq.png + :width: 400 + +The figure above shows a parallel application of 12 processes producing a 2D array. Each process has a 2D array locally +and the output is created by placing them into a 4x3 pattern. A reading application's individual process then can read +any subsection of the entire global array. In the figure, a 6 process application decomposes the array in a 3x2 pattern +and each process reads a 2D array whose content comes from multiple producer processes. + +The figure hopefully helps to understand the basic concept but it can be also misleading if it suggests limitations that +are not there. Global Array is simply a boundary in N-dimensional space where processes can place their blocks of data. +In the global space: + +1. one process can place multiple blocks + + .. image:: https://imgur.com/Pb1s03h.png + :width: 400 + +2. does NOT need to be fully covered by the blocks + + .. image:: https://imgur.com/qJBXYcQ.png + :width: 400 + + * at reading, unfilled positions will not change the allocated memory + +3. blocks can overlap + + .. image:: https://imgur.com/GA59lZ2.png + :width: 300 + * the reader will get values in an overlapping position from one of the block but there is no control over from which + block +4. each process can put a different size of block, or put multiple blocks of different sizes + +5. some process may not contribute anything to the global array + +Over multiple output steps + +1. the processes CAN change the size (and number) of blocks in the array + + * E.g. atom table: global size is fixed but atoms wander around processes, so their block size is changing + + .. image:: https://imgur.com/DorjG2q.png + :width: 400 + +2. the global dimensions CAN change over output steps + + * but then you cannot read multiple steps at once + * E.g. particle table size changes due to particles disappearing or appearing + + .. image:: https://imgur.com/nkuHeVX.png + :width: 400 + + +Limitations of the ADIOS global array concept + +1. Indexing starts from 0 +2. Cyclic data patterns are not supported; only blocks can be written or read +3. If Some blocks may fully or partially fall outside of the global boundary, the reader will not be able to read those + parts + +.. note:: + Technically, the content of the individual blocks is kept in the BP format (but not in HDF5 format) and in staging. + If you really, really want to retrieve all the blocks, you need to handle this array as a Local Array and read the + blocks one by one. From d8a70b3193ef85a7f188813ce2b9b6ebf91e0836 Mon Sep 17 00:00:00 2001 From: pnorbert Date: Tue, 5 Dec 2023 19:32:36 -0500 Subject: [PATCH 14/18] Use adios2_mode_readRandomAccess in matlab open to make it work for BP5 (#3956) --- bindings/Matlab/adiosopenc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/Matlab/adiosopenc.c b/bindings/Matlab/adiosopenc.c index b9b4c1a88a..07c4188d36 100644 --- a/bindings/Matlab/adiosopenc.c +++ b/bindings/Matlab/adiosopenc.c @@ -142,7 +142,7 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]) /* Open ADIOS file now and get variables and attributes */ adiosobj = adios2_init_serial(); group = adios2_declare_io(adiosobj, "matlabiogroup"); // name is arbitrary - fp = adios2_open(group, fname, adios2_mode_read); + fp = adios2_open(group, fname, adios2_mode_readRandomAccess); if (fp == NULL) { mexErrMsgIdAndTxt("MATLAB:adiosopenc:open", "Opening the file failed\n"); From 1e5993ef055f0615855c70d1922ae39e2c42b876 Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Tue, 5 Dec 2023 18:43:56 -0700 Subject: [PATCH 15/18] ci: fix path to lsan suppressions, fix broken gh status post --- .circleci/config.yml | 4 ++++ scripts/ci/circle/postCDashStatus.sh | 2 +- scripts/ci/cmake/ci-fedora-asan.cmake | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d95cac9fa1..75044102ac 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,6 +2,10 @@ defaults: &defaults steps: - checkout: path: source + - run: + # https://discuss.circleci.com/t/circle-working-directory-doesnt-expand/17007 + name: "Fix CIRCLE_WORKING_DIRECTORY" + command: echo 'CIRCLE_WORKING_DIRECTORY="${CIRCLE_WORKING_DIRECTORY/#\~/$HOME}"' >> $BASH_ENV - run: name: CDash command: bash source/scripts/ci/circle/postCDashStatus.sh diff --git a/scripts/ci/circle/postCDashStatus.sh b/scripts/ci/circle/postCDashStatus.sh index 031ae912cc..78ec1e2228 100755 --- a/scripts/ci/circle/postCDashStatus.sh +++ b/scripts/ci/circle/postCDashStatus.sh @@ -17,7 +17,7 @@ build_status_body() { EOF } -PYTHON_SCRIPT="${SOURCE_DIR}/scripts/ci/findStatus.py" +PYTHON_SCRIPT="${SOURCE_DIR}/scripts/ci/circle/findStatus.py" curl -u "${USER}:${TOKEN}" "${API_BASE}/commits/${COMMIT}/statuses" | python3 "${PYTHON_SCRIPT}" --context ${CDASH_STATUS_CONTEXT} exit_status=$? if [ "$exit_status" -ne 0 ] diff --git a/scripts/ci/cmake/ci-fedora-asan.cmake b/scripts/ci/cmake/ci-fedora-asan.cmake index 8651193fd8..8b7e6dcd65 100644 --- a/scripts/ci/cmake/ci-fedora-asan.cmake +++ b/scripts/ci/cmake/ci-fedora-asan.cmake @@ -4,7 +4,7 @@ set(ENV{CC} clang) set(ENV{CXX} clang++) set(ASAN_FLAGS "-fsanitize=address -fno-omit-frame-pointer -pthread -mllvm -asan-use-private-alias=1 -Wno-unused-command-line-argument") set(ENV{ASAN_OPTIONS} "use_odr_indicator=1") -set(ENV{LSAN_OPTIONS} "suppressions=${CMAKE_SOURCE_DIR}/thirdparty/perfstubs/perfstubs.supp") +set(ENV{LSAN_OPTIONS} "suppressions=$ENV{CI_SOURCE_DIR}/thirdparty/perfstubs/perfstubs.supp") set(ENV{CFLAGS} "${ASAN_FLAGS}") set(ENV{CXXFLAGS} "${ASAN_FLAGS}") From 42e062b1e812afcdd12d683ef1f8a9ff890df505 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 6 Dec 2023 14:20:08 +0100 Subject: [PATCH 16/18] Adapt libfabric dataplane of SST to Cray CXI provider (#3672) * Support detection of CXI in CMake * Extended error detection in libfabric dataplane * Support for FI_MR_ENDPOINT * Support for FI_MR_VIRT_ADDR == 0 * Full support for CXI provider * clang-format fixes * Fix behavior when using FI_MR_BASIC FI_MR_BASIC is special and mode bits cannot be compared against it. --- cmake/DetectOptions.cmake | 8 + cmake/check_libfabric_cxi.c | 5 + source/adios2/toolkit/sst/CMakeLists.txt | 1 + source/adios2/toolkit/sst/dp/rdma_dp.c | 683 +++++++++++++++++++++-- 4 files changed, 635 insertions(+), 62 deletions(-) create mode 100644 cmake/check_libfabric_cxi.c diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index efecb57624..d66e096bda 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -450,6 +450,14 @@ if(ADIOS2_USE_SST AND NOT WIN32) if(CrayDRC_FOUND) set(ADIOS2_SST_HAVE_CRAY_DRC TRUE) endif() + + try_compile(ADIOS2_SST_HAVE_CRAY_CXI + ${ADIOS2_BINARY_DIR}/check_libfabric_cxi + ${ADIOS2_SOURCE_DIR}/cmake/check_libfabric_cxi.c + CMAKE_FLAGS + "-DINCLUDE_DIRECTORIES=${LIBFABRIC_INCLUDE_DIRS}" + "-DLINK_DIRECTORIES=${LIBFABRIC_LIBRARIES}") + message(STATUS "Libfabric support for the HPE CXI provider: ${ADIOS2_SST_HAVE_CRAY_CXI}") endif() if(ADIOS2_HAVE_MPI) set(CMAKE_REQUIRED_LIBRARIES "MPI::MPI_C;Threads::Threads") diff --git a/cmake/check_libfabric_cxi.c b/cmake/check_libfabric_cxi.c new file mode 100644 index 0000000000..c2f2c80878 --- /dev/null +++ b/cmake/check_libfabric_cxi.c @@ -0,0 +1,5 @@ +#include +#include +#include + +int main() {} diff --git a/source/adios2/toolkit/sst/CMakeLists.txt b/source/adios2/toolkit/sst/CMakeLists.txt index 070c8f41a1..c77fc0459b 100644 --- a/source/adios2/toolkit/sst/CMakeLists.txt +++ b/source/adios2/toolkit/sst/CMakeLists.txt @@ -65,6 +65,7 @@ set(SST_CONFIG_OPTS UCX FI_GNI CRAY_DRC + CRAY_CXI NVStream MPI ) diff --git a/source/adios2/toolkit/sst/dp/rdma_dp.c b/source/adios2/toolkit/sst/dp/rdma_dp.c index e61c6f5f3f..e3800f4ff0 100644 --- a/source/adios2/toolkit/sst/dp/rdma_dp.c +++ b/source/adios2/toolkit/sst/dp/rdma_dp.c @@ -19,6 +19,13 @@ #include #include +#ifdef SST_HAVE_CRAY_CXI +#include +// This comment prevents clang-format from reordering these includes. +// The CXI extension header requires the bool header, but does not include it on its own. +#include +#endif + #if defined(__has_feature) #if __has_feature(thread_sanitizer) #define NO_SANITIZE_THREAD __attribute__((no_sanitize("thread"))) @@ -53,6 +60,66 @@ static pthread_mutex_t fabric_mutex = PTHREAD_MUTEX_INITIALIZER; pthread_mutex_t wsr_mutex = PTHREAD_MUTEX_INITIALIZER; pthread_mutex_t ts_mutex = PTHREAD_MUTEX_INITIALIZER; +/* + * Wrapper for fi_mr_reg() with additional parameters endpoint and mr_mode. + * If mr_mode includes FI_MR_ENDPOINT, the memory region must be bound to the + * endpoint and enabled before use. + */ +int sst_fi_mr_reg( + /* first two parameters for verbose logging */ + CP_Services Svcs, void *CP_Stream, + /* regular fi_mir_reg() parameters*/ + struct fid_domain *domain, const void *buf, size_t len, uint64_t acs, uint64_t offset, + uint64_t requested_key, uint64_t flags, struct fid_mr **mr, void *context, + /* additional parameters for binding the mr to the endpoint*/ + struct fid_ep *endpoint, int mr_mode) +{ + int res = fi_mr_reg(domain, buf, len, acs, offset, requested_key, flags, mr, context); + int is_mr_endpoint = (mr_mode & FI_MR_ENDPOINT) != 0; + if (!is_mr_endpoint) + { + return res; + } + if (res != FI_SUCCESS) + { + Svcs->verbose(CP_Stream, DPCriticalVerbose, "fi_mr_reg failed with %ul (%s)\n", res, + fi_strerror(res)); + return res; + } + + /* + * When the domain_attr->mr_mode includes FI_MR_ENDPOINT, the memory region + * needs to be bound to the endpoint and explicitly enabled after that. + */ + res = fi_mr_bind(*mr, &endpoint->fid, 0); + if (res != FI_SUCCESS) + { + Svcs->verbose(CP_Stream, DPCriticalVerbose, "fi_mr_bind failed with %ul (%s)\n", res, + fi_strerror(res)); + return res; + } + res = fi_mr_enable(*mr); + if (res != FI_SUCCESS) + { + Svcs->verbose(CP_Stream, DPCriticalVerbose, "fi_mr_enable failed with %ul (%s)\n", res, + fi_strerror(res)); + return res; + } + return res; +} + +/* + * Simple wrapper to create a log entry upon failing fi_*() function calls. + */ +int guard_fi_return(int code, CP_Services Svcs, CManager cm, char const *msg) +{ + if (code != FI_SUCCESS) + { + Svcs->verbose(cm, DPCriticalVerbose, "%s: %s (%lu)\n", msg, fi_strerror(code), code); + } + return code; +} + struct fabric_state { struct fi_context *ctx; @@ -60,6 +127,7 @@ struct fabric_state struct fi_info *info; // struct fi_info *linfo; int local_mr_req; + int mr_virt_addr; /* Stores if the mr_mode includes FI_MR_VIRT_ADDR */ int rx_cq_data; size_t addr_len; size_t msg_prefix_size; @@ -69,6 +137,9 @@ struct fabric_state struct fid_cq *cq_signal; struct fid_av *av; pthread_t listener; +#ifdef SST_HAVE_CRAY_CXI + struct cxi_auth_key *cxi_auth_key; +#endif #ifdef SST_HAVE_CRAY_DRC drc_info_handle_t drc_info; uint32_t credential; @@ -112,13 +183,24 @@ struct fabric_state * plane would replace one or both of these with RDMA functionality. */ +static char const *get_preferred_domain(struct _SstParams *Params) +{ + if (Params->DataInterface) + { + return Params->DataInterface; + } + else + { + return getenv("FABRIC_IFACE"); + } +} + static void init_fabric(struct fabric_state *fabric, struct _SstParams *Params, CP_Services Svcs, - void *CP_Stream) + void *CP_Stream, char const *ifname) { struct fi_info *hints, *info, *originfo, *useinfo; struct fi_av_attr av_attr = {FI_AV_UNSPEC}; struct fi_cq_attr cq_attr = {0}; - char *ifname; int result; hints = fi_allocinfo(); @@ -126,24 +208,76 @@ static void init_fabric(struct fabric_state *fabric, struct _SstParams *Params, FI_MSG | FI_SEND | FI_RECV | FI_REMOTE_READ | FI_REMOTE_WRITE | FI_RMA | FI_READ | FI_WRITE; hints->mode = FI_CONTEXT | FI_LOCAL_MR | FI_CONTEXT2 | FI_MSG_PREFIX | FI_ASYNC_IOV | FI_RX_CQ_DATA; - hints->domain_attr->mr_mode = FI_MR_BASIC; - hints->domain_attr->control_progress = FI_PROGRESS_AUTO; - hints->domain_attr->data_progress = FI_PROGRESS_AUTO; hints->ep_attr->type = FI_EP_RDM; - if (Params->DataInterface) + uint32_t fi_version; +#ifdef SST_HAVE_CRAY_CXI + if (fabric->cxi_auth_key) { - ifname = Params->DataInterface; + fi_version = FI_VERSION(1, 11); + + hints->domain_attr->mr_mode = FI_MR_ENDPOINT; + hints->domain_attr->control_progress = FI_PROGRESS_MANUAL; + hints->domain_attr->data_progress = FI_PROGRESS_MANUAL; + + // Authentication is needed + // TODO: the first ID in SLINGSHOT_SVC_IDS is chosen, but we should + // rather choose the one corresponding with the FABRIC_IFACE + // example: + // SLINGSHOT_SVC_IDS=5,5,5,5 + // SLINGSHOT_VNIS=1310,1271 + // SLINGSHOT_DEVICES=cxi0,cxi1,cxi2,cxi3 + // FABRIC_IFACE=cxi2 (user specified) + + hints->ep_attr->auth_key = malloc(sizeof(struct cxi_auth_key)); + memcpy(hints->ep_attr->auth_key, fabric->cxi_auth_key, sizeof(struct cxi_auth_key)); + hints->ep_attr->auth_key_size = sizeof(struct cxi_auth_key); + + hints->domain_attr->auth_key = malloc(sizeof(struct cxi_auth_key)); + memcpy(hints->domain_attr->auth_key, fabric->cxi_auth_key, sizeof(struct cxi_auth_key)); + hints->domain_attr->auth_key_size = sizeof(struct cxi_auth_key); } else { - ifname = getenv("FABRIC_IFACE"); + fi_version = FI_VERSION(1, 5); + + hints->domain_attr->mr_mode = FI_MR_BASIC; + hints->domain_attr->control_progress = FI_PROGRESS_AUTO; + hints->domain_attr->data_progress = FI_PROGRESS_AUTO; + } +#else + fi_version = FI_VERSION(1, 5); + + // Alternatively, one could set mr_mode to + // FI_MR_VIRT_ADDR | FI_MR_ALLOCATED | FI_MR_PROV_KEY | FI_MR_LOCAL + // here. These flags are equivalent to FI_MR_BASIC, but unlike basic + // registration, providers are not forced to keep those flags when they + // think that not using the flags is better. + // The RDMA DP is able to deal with this appropriately, and does so right + // before calling fi_fabric() further below in this function. + // The main reason for keeping FI_MR_BASIC here is backward compatibility. + hints->domain_attr->mr_mode = FI_MR_BASIC; + hints->domain_attr->control_progress = FI_PROGRESS_AUTO; + hints->domain_attr->data_progress = FI_PROGRESS_AUTO; +#endif + + /* + * ifname is passed as a function parameter of init_fabric() if + * a provider-specific key was configured and sent to the reader. + * Since the key is generally domain-specific, we must use that one in this + * case. + * The preferred domain is already considered upon key configuration, + * so this is fine. + */ + if (!ifname) + { + ifname = get_preferred_domain(Params); } fabric->info = NULL; pthread_mutex_lock(&fabric_mutex); - fi_getinfo(FI_VERSION(1, 5), NULL, NULL, 0, hints, &info); + fi_getinfo(fi_version, NULL, NULL, 0, hints, &info); pthread_mutex_unlock(&fabric_mutex); if (!info) { @@ -167,7 +301,8 @@ static void init_fabric(struct fabric_state *fabric, struct _SstParams *Params, break; } if ((((strcmp(prov_name, "verbs") == 0) && info->src_addr) || - (strcmp(prov_name, "gni") == 0) || (strcmp(prov_name, "psm2") == 0)) && + (strcmp(prov_name, "gni") == 0) || (strcmp(prov_name, "psm2") == 0) || + (strcmp(prov_name, "cxi") == 0)) && (!useinfo || !ifname || (strcmp(useinfo->domain_attr->name, ifname) != 0))) { Svcs->verbose(CP_Stream, DPTraceVerbose, @@ -177,7 +312,7 @@ static void init_fabric(struct fabric_state *fabric, struct _SstParams *Params, useinfo = info; } else if (((strstr(prov_name, "verbs") && info->src_addr) || strstr(prov_name, "gni") || - strstr(prov_name, "psm2")) && + strstr(prov_name, "psm2") || strstr(prov_name, "cxi")) && !useinfo) { Svcs->verbose(CP_Stream, DPTraceVerbose, @@ -253,7 +388,30 @@ static void init_fabric(struct fabric_state *fabric, struct _SstParams *Params, fabric->addr_len = info->src_addrlen; - info->domain_attr->mr_mode = FI_MR_BASIC; + /* + * The libfabric data-plane of SST was originally programmed to use + * FI_MR_BASIC as mr_mode, which is equivalent to + * FI_MR_VIRT_ADDR | FI_MR_ALLOCATED | FI_MR_PROV_KEY | FI_MR_LOCAL. + * + * However, HPE's CXI provider requires two changes to that: + * (1) It does not support FI_MR_VIRT_ADDR. + * (2) It requires use of FI_MR_ENDPOINT. + * + * So we propagate the bit value currently contained in the mr_mode + * for these flags. + */ + if (info->domain_attr->mr_mode != FI_MR_BASIC) + { + info->domain_attr->mr_mode = FI_MR_ALLOCATED | FI_MR_PROV_KEY | FI_MR_LOCAL | + (FI_MR_ENDPOINT & info->domain_attr->mr_mode) | + (FI_MR_VIRT_ADDR & info->domain_attr->mr_mode); + fabric->mr_virt_addr = info->domain_attr->mr_mode & FI_MR_VIRT_ADDR ? 1 : 0; + } + else + { + fabric->mr_virt_addr = 1; + } + #ifdef SST_HAVE_CRAY_DRC if (strstr(info->fabric_attr->prov_name, "gni") && fabric->auth_key) { @@ -411,6 +569,12 @@ static void fini_fabric(struct fabric_state *fabric, CP_Services Svcs, void *CP_ free(fabric->ctx); } +#ifdef SST_HAVE_CRAY_CXI + if (fabric->cxi_auth_key) + { + free(fabric->cxi_auth_key); + } +#endif #ifdef SST_HAVE_CRAY_DRC if (Fabric->auth_key) { @@ -599,6 +763,191 @@ static TimestepList GetStep(Rdma_WS_Stream Stream, long Timestep) return (Step); } +#ifdef SST_HAVE_CRAY_CXI +static int get_cxi_auth_key_from_env(CP_Services Svcs, void *CP_Stream, struct _SstParams *Params, + struct cxi_auth_key *key, char **used_device) +{ + int vni, first_vni, second_vni, svc_id; + + // Just some safety against faulty strings in string processing. + size_t const no_infinite_loops = 10000; + + // struct cxi_auth_key { + // /* The CXI service assigned to the Domain and Endpoints. A CXI + // service + // * is associated with a set of local resource limits, VNIs, and + // Traffic + // * Classes. + // * + // * The svc_id used by an OFI Domain must match all Endpoints belonging + // * to the Domain. + // */ + // uint32_t svc_id; + + // /* The Virtual Network ID (VNI) assigned to the Endpoint. Two + // Endpoints + // * must use the same VNI in order to communicate. + // * + // * Note that while the CXI service may define one or more VNIs which a + // * process can access, an Endpoint is assigned to only one. + // */ + // uint16_t vni; + // }; + + // typical value SLINGSHOT_DEVICES=cxi0,cxi1,cxi2,cxi3 + char const *slingshot_devices = getenv("SLINGSHOT_DEVICES"); + char const *preferred_device = get_preferred_domain(Params); + + /* + * In the following loop, find out if the preferred_device is found within + * the slingshot_devices. + * If the preferred_device is NULL, just pick the first. + * Upon success, modifies the output parameter used_device and stores + * the retrieved device index. + */ + size_t device_index = 0; + for (size_t no_infinite_loop_counter = 0;; ++device_index, ++no_infinite_loop_counter) + { + if (no_infinite_loop_counter > no_infinite_loops) + { + return EXIT_FAILURE; + } + + // Are we at the end of the environment variable? + int found_end = 0; + + // Find out the length of the current item in slingshot_devices. + size_t find_end_of_current_string = 0; + for (size_t no_infinite_loop_inner_counter = 0;; + ++find_end_of_current_string, ++no_infinite_loop_inner_counter) + { + if (no_infinite_loop_inner_counter > no_infinite_loops) + { + return EXIT_FAILURE; + } + + switch (slingshot_devices[find_end_of_current_string]) + { + case '\0': + found_end = 1; + goto break_first_loop; + case ',': + goto break_first_loop; + default: + break; + } + } + break_first_loop:; + int use_this_device = !preferred_device || (strncmp(preferred_device, slingshot_devices, + find_end_of_current_string) == 0); + if (use_this_device) + { + char *construct_used_device = malloc(find_end_of_current_string + 1); + memcpy(construct_used_device, slingshot_devices, find_end_of_current_string); + construct_used_device[find_end_of_current_string] = '\0'; + *used_device = construct_used_device; + break; + } + else if (found_end) + { + return EXIT_FAILURE; + } + else + { + // go to next iteration + slingshot_devices += find_end_of_current_string + 1; + } + } + + Svcs->verbose(CP_Stream, DPTraceVerbose, "Found device %s at index %zu\n", *used_device, + device_index); + + // typical value SLINGSHOT_VNIS=4576,4530 + char const *vni_env_str = getenv("SLINGSHOT_VNIS"); + if (!vni_env_str) + { + return EXIT_FAILURE; + } + + // typical value SLINGSHOT_SVC_IDS=5,5,5,5 + char const *svc_ids_env_str = getenv("SLINGSHOT_SVC_IDS"); + if (!svc_ids_env_str) + { + return EXIT_FAILURE; + } + + { + int num_vnis = sscanf(vni_env_str, "%d,%d", &first_vni, &second_vni); + switch (num_vnis) + { + // first VNI is the subjob's VNI + case 1: + Svcs->verbose(CP_Stream, DPTraceVerbose, "Using first vni.\n"); + vni = first_vni; + break; + // if present, the second VNI is the containing job's VNI + // the first VNI belongs to the subjob + case 2: + Svcs->verbose(CP_Stream, DPTraceVerbose, "Using second vni.\n"); + vni = second_vni; + break; + default: + return EXIT_FAILURE; + } + } + + { + // Pick the service ID according to the device_index found above. + for (size_t svc_id_index = 0; svc_id_index < device_index; ++svc_id_index) + { + for (size_t no_infinite_loop_counter = 0;; ++no_infinite_loop_counter) + { + if (no_infinite_loop_counter > no_infinite_loops) + { + return EXIT_FAILURE; + } + + switch (*(svc_ids_env_str++)) + { + case ',': + goto break_second_loop; + case '\0': + return EXIT_FAILURE; + default: + continue; + } + } + break_second_loop:; + } + + int num_svc_ids = sscanf(svc_ids_env_str, "%d", &svc_id); + switch (num_svc_ids) + { + case 1: + break; + default: + return EXIT_FAILURE; + } + } + + key->vni = vni; + key->svc_id = svc_id; + + return EXIT_SUCCESS; +} + +static int get_cxi_auth_key_from_writer(struct cxi_auth_key *key, attr_list WriterContact) +{ + long vni; + if (!get_long_attr(WriterContact, attr_atom_from_string("vni"), &vni)) + { + return EXIT_FAILURE; + } + key->vni = (uint16_t)vni; + return EXIT_SUCCESS; +} +#endif + static DP_RS_Stream RdmaInitReader(CP_Services Svcs, void *CP_Stream, void **ReaderContactInfoPtr, struct _SstParams *Params, attr_list WriterContact, SstStats Stats) @@ -645,6 +994,38 @@ static DP_RS_Stream RdmaInitReader(CP_Services Svcs, void *CP_Stream, void **Rea Stream->PreloadAvail = 0; } + char *required_device = NULL; +#ifdef SST_HAVE_CRAY_CXI + struct + { + struct cxi_auth_key key; + int valid; + } tagged_key; + + /* + * The svc_id of the key must match the device that this particular reader + * connects with. + * The vni (virtual network ID) must be the same across all communicating + * instances (get this from the writer). + */ + + tagged_key.valid = + get_cxi_auth_key_from_env(Svcs, CP_Stream, Params, &tagged_key.key, &required_device); + + if (tagged_key.valid == EXIT_SUCCESS && + get_cxi_auth_key_from_writer(&tagged_key.key, WriterContact) == EXIT_SUCCESS) + { + Svcs->verbose(CP_Stream, DPSummaryVerbose, "Reader found CXI auth key: %d %d\n", + tagged_key.key.vni, tagged_key.key.svc_id); + Stream->Fabric->cxi_auth_key = calloc(1, sizeof(struct cxi_auth_key)); + memcpy(Stream->Fabric->cxi_auth_key, &tagged_key.key, sizeof(struct cxi_auth_key)); + } + else + { + Svcs->verbose(CP_Stream, DPSummaryVerbose, "Reader found no CXI auth key\n"); + } +#endif + #ifdef SST_HAVE_CRAY_DRC int attr_cred, try_left, rc; if (!get_int_attr(WriterContact, attr_atom_from_string("RDMA_DRC_KEY"), &attr_cred)) @@ -675,7 +1056,11 @@ static DP_RS_Stream RdmaInitReader(CP_Services Svcs, void *CP_Stream, void **Rea #endif /* SST_HAVE_CRAY_DRC */ - init_fabric(Stream->Fabric, Stream->Params, Svcs, CP_Stream); + init_fabric(Stream->Fabric, Stream->Params, Svcs, CP_Stream, required_device); + if (required_device) + { + free(required_device); + } if (!Fabric->info) { Svcs->verbose(CP_Stream, DPCriticalVerbose, "Could not find a valid transport fabric.\n"); @@ -684,7 +1069,12 @@ static DP_RS_Stream RdmaInitReader(CP_Services Svcs, void *CP_Stream, void **Rea ContactInfo->Length = Fabric->info->src_addrlen; ContactInfo->Address = malloc(ContactInfo->Length); - fi_getname((fid_t)Fabric->signal, ContactInfo->Address, &ContactInfo->Length); + if (guard_fi_return( + fi_getname((fid_t)Fabric->signal, ContactInfo->Address, &ContactInfo->Length), Svcs, + CP_Stream, "[RdmaInitReader] fi_getname() failed with:") != FI_SUCCESS) + { + return NULL; + } Stream->PreloadStep = -1; Stream->ContactInfo = ContactInfo; @@ -775,6 +1165,42 @@ static DP_WS_Stream RdmaInitWriter(CP_Services Svcs, void *CP_Stream, struct _Ss Stream->Fabric = calloc(1, sizeof(struct fabric_state)); Fabric = Stream->Fabric; + + char *required_device = NULL; +#ifdef SST_HAVE_CRAY_CXI + struct + { + struct cxi_auth_key key; + int valid; + } tagged_key; + + /* + * The svc_id of the key must match the device that this particular writer + * connects with. + * The vni (virtual network ID) must be the same across all communicating + * instances (use the one seen by rank 0). + */ + tagged_key.valid = + get_cxi_auth_key_from_env(Svcs, CP_Stream, Params, &tagged_key.key, &required_device); + + // Ensure that all writers use the same virtual network ID + SMPI_Bcast(&tagged_key.key.vni, sizeof(tagged_key.key.vni), SMPI_BYTE, 0, comm); + + if (tagged_key.valid == EXIT_SUCCESS) + { + Svcs->verbose(CP_Stream, DPSummaryVerbose, "Writer found CXI auth key: %d %d\n", + tagged_key.key.vni, tagged_key.key.svc_id); + + set_long_attr(DPAttrs, attr_atom_from_string("vni"), tagged_key.key.vni); + Stream->Fabric->cxi_auth_key = calloc(1, sizeof(struct cxi_auth_key)); + memcpy(Stream->Fabric->cxi_auth_key, &tagged_key.key, sizeof(struct cxi_auth_key)); + } + else + { + Svcs->verbose(CP_Stream, DPSummaryVerbose, "Writer found no CXI auth key"); + } +#endif + #ifdef SST_HAVE_CRAY_DRC int try_left, rc; if (Stream->Rank == 0) @@ -818,7 +1244,11 @@ static DP_WS_Stream RdmaInitWriter(CP_Services Svcs, void *CP_Stream, struct _Ss set_long_attr(DPAttrs, attr_atom_from_string("RDMA_DRC_CRED"), attr_cred); #endif /* SST_HAVE_CRAY_DRC */ - init_fabric(Stream->Fabric, Params, Svcs, CP_Stream); + init_fabric(Stream->Fabric, Params, Svcs, CP_Stream, required_device); + if (required_device) + { + free(required_device); + } Fabric = Stream->Fabric; if (!Fabric->info) { @@ -872,8 +1302,15 @@ static DP_WSR_Stream RdmaInitWriterPerReader(CP_Services Svcs, DP_WS_Stream WS_S for (i = 0; i < readerCohortSize; i++) { - fi_av_insert(Fabric->av, providedReaderInfo[i]->Address, 1, &WSR_Stream->ReaderAddr[i], 0, - NULL); + if (fi_av_insert(Fabric->av, providedReaderInfo[i]->Address, 1, &WSR_Stream->ReaderAddr[i], + 0, NULL) < 1) + { + + Svcs->verbose(WS_Stream->CP_Stream, DPCriticalVerbose, + "[RdmaInitWRiterPerReader] Failed inserting address " + "into vector\n"); + return NULL; + } Svcs->verbose(WS_Stream->CP_Stream, DPTraceVerbose, "Received contact info for RS_Stream %p, WSR Rank %d\n", providedReaderInfo[i]->RS_Stream, i); @@ -895,13 +1332,20 @@ static DP_WSR_Stream RdmaInitWriterPerReader(CP_Services Svcs, DP_WS_Stream WS_S ContactInfo->Length = Fabric->info->src_addrlen; ContactInfo->Address = malloc(ContactInfo->Length); - fi_getname((fid_t)Fabric->signal, ContactInfo->Address, &ContactInfo->Length); + if (guard_fi_return( + fi_getname((fid_t)Fabric->signal, ContactInfo->Address, &ContactInfo->Length), Svcs, + WS_Stream->CP_Stream, + "[RdmaInitWriterPerReader] fi_getname() failed with") != FI_SUCCESS) + { + return NULL; + } ReaderRollHandle = &ContactInfo->ReaderRollHandle; ReaderRollHandle->Block = calloc(readerCohortSize, sizeof(struct _RdmaBuffer)); - fi_mr_reg(Fabric->domain, ReaderRollHandle->Block, - readerCohortSize * sizeof(struct _RdmaBuffer), FI_REMOTE_WRITE, 0, 0, 0, - &WSR_Stream->rrmr, Fabric->ctx); + sst_fi_mr_reg(Svcs, WS_Stream->CP_Stream, Fabric->domain, ReaderRollHandle->Block, + readerCohortSize * sizeof(struct _RdmaBuffer), FI_REMOTE_WRITE, 0, 0, 0, + &WSR_Stream->rrmr, Fabric->ctx, Fabric->signal, + Fabric->info->domain_attr->mr_mode); ReaderRollHandle->Key = fi_mr_key(WSR_Stream->rrmr); WSR_Stream->WriterContactInfo = ContactInfo; @@ -943,8 +1387,15 @@ static void RdmaProvideWriterDataToReader(CP_Services Svcs, DP_RS_Stream RS_Stre for (int i = 0; i < writerCohortSize; i++) { RS_Stream->WriterContactInfo[i].WS_Stream = providedWriterInfo[i]->WS_Stream; - fi_av_insert(Fabric->av, providedWriterInfo[i]->Address, 1, &RS_Stream->WriterAddr[i], 0, - NULL); + if (fi_av_insert(Fabric->av, providedWriterInfo[i]->Address, 1, &RS_Stream->WriterAddr[i], + 0, NULL) < 1) + { + Svcs->verbose(RS_Stream->CP_Stream, DPCriticalVerbose, + "[RdmaProvideWriterDataToReader] " + "Failed inserting address " + "into vector\n"); + return; + } RS_Stream->WriterRoll[i] = providedWriterInfo[i]->ReaderRollHandle; Svcs->verbose(RS_Stream->CP_Stream, DPTraceVerbose, "Received contact info for WS_stream %p, WSR Rank %d\n", @@ -1032,11 +1483,21 @@ static ssize_t PostRead(CP_Services Svcs, Rdma_RS_Stream RS_Stream, int Rank, lo if (Fabric->local_mr_req) { // register dest buffer - fi_mr_reg(Fabric->domain, Buffer, Length, FI_READ, 0, 0, 0, &ret->LocalMR, Fabric->ctx); + sst_fi_mr_reg(Svcs, RS_Stream->CP_Stream, Fabric->domain, Buffer, Length, FI_READ, 0, 0, 0, + &ret->LocalMR, Fabric->ctx, Fabric->signal, + Fabric->info->domain_attr->mr_mode); LocalDesc = fi_mr_desc(ret->LocalMR); } - Addr = Info->Block + Offset; + if (Fabric->mr_virt_addr) + { + Addr = Info->Block + Offset; + } + else + { + Addr = NULL; + Addr += Offset; + } Svcs->verbose(RS_Stream->CP_Stream, DPTraceVerbose, "Remote read target is Rank %d (Offset = %zi, Length = %zi)\n", Rank, Offset, @@ -1203,8 +1664,14 @@ static int DoPushWait(CP_Services Svcs, Rdma_RS_Stream Stream, RdmaCompletionHan rc = fi_cq_sread(Fabric->cq_signal, (void *)(&CQEntry), 1, NULL, -1); if (rc < 1) { - Svcs->verbose(Stream->CP_Stream, DPCriticalVerbose, - "failure while waiting for completions (%d).\n", rc); + struct fi_cq_err_entry error; + fi_cq_readerr(Fabric->cq_signal, &error, 0); + Svcs->verbose( + Stream->CP_Stream, DPCriticalVerbose, + "failure while waiting for completions inside " + "DoPushWait() (%d (%s - %s)).\n", + rc, fi_strerror(error.err), + fi_cq_strerror(Fabric->cq_signal, error.err, error.err_data, NULL, error.len)); return 0; } else if (CQEntry.flags & FI_REMOTE_CQ_DATA) @@ -1276,8 +1743,14 @@ static int WaitForAnyPull(CP_Services Svcs, Rdma_RS_Stream Stream) rc = fi_cq_sread(Fabric->cq_signal, (void *)(&CQEntry), 1, NULL, -1); if (rc < 1) { - Svcs->verbose(Stream->CP_Stream, DPCriticalVerbose, - "failure while waiting for completions (%d).\n", rc); + struct fi_cq_err_entry error; + fi_cq_readerr(Fabric->cq_signal, &error, 0); + Svcs->verbose( + Stream->CP_Stream, DPCriticalVerbose, + "failure while waiting for completions inside " + "WaitForAnyPull() (%d (%s - %s)).\n", + rc, fi_strerror(error.err), + fi_cq_strerror(Fabric->cq_signal, error.err, error.err_data, NULL, error.len)); return 0; } else @@ -1345,8 +1818,9 @@ static void RdmaProvideTimestep(CP_Services Svcs, DP_WS_Stream Stream_v, struct Entry->BufferSlot = -1; Entry->Desc = NULL; - fi_mr_reg(Fabric->domain, Data->block, Data->DataSize, FI_WRITE | FI_REMOTE_READ, 0, 0, 0, - &Entry->mr, Fabric->ctx); + sst_fi_mr_reg(Svcs, Stream->CP_Stream, Fabric->domain, Data->block, Data->DataSize, + FI_WRITE | FI_REMOTE_READ, 0, 0, 0, &Entry->mr, Fabric->ctx, Fabric->signal, + Fabric->info->domain_attr->mr_mode); Entry->Key = fi_mr_key(Entry->mr); if (Fabric->local_mr_req) { @@ -1608,7 +2082,7 @@ static struct _CP_DP_Interface RdmaDPInterface = {0}; static int RdmaGetPriority(CP_Services Svcs, void *CP_Stream, struct _SstParams *Params) { struct fi_info *hints, *info, *originfo; - char *ifname; + char const *ifname; char *forkunsafe; int Ret = -1; @@ -1617,20 +2091,41 @@ static int RdmaGetPriority(CP_Services Svcs, void *CP_Stream, struct _SstParams FI_MSG | FI_SEND | FI_RECV | FI_REMOTE_READ | FI_REMOTE_WRITE | FI_RMA | FI_READ | FI_WRITE; hints->mode = FI_CONTEXT | FI_LOCAL_MR | FI_CONTEXT2 | FI_MSG_PREFIX | FI_ASYNC_IOV | FI_RX_CQ_DATA; - hints->domain_attr->mr_mode = FI_MR_BASIC; - hints->domain_attr->control_progress = FI_PROGRESS_AUTO; - hints->domain_attr->data_progress = FI_PROGRESS_AUTO; hints->ep_attr->type = FI_EP_RDM; - if (Params->DataInterface) + char const *vni_env_str = getenv("SLINGSHOT_VNIS"); + + uint32_t fi_version; + if (vni_env_str) { - ifname = Params->DataInterface; + // try fishing for the CXI provider + Svcs->verbose(CP_Stream, DPSummaryVerbose, + "RDMA Dataplane trying to check for an available CXI " + "provider since environment variable SLINGSHOT_VNIS is " + "defined (value: '%s').\n", + vni_env_str); + fi_version = FI_VERSION(1, 11); + + hints->domain_attr->mr_mode = FI_MR_ENDPOINT; + hints->domain_attr->control_progress = FI_PROGRESS_MANUAL; + hints->domain_attr->data_progress = FI_PROGRESS_MANUAL; } else { - ifname = getenv("FABRIC_IFACE"); + Svcs->verbose(CP_Stream, DPSummaryVerbose, + "RDMA Dataplane trying to check for an available non-CXI " + "provider since environment variable SLINGSHOT_VNIS is " + "not defined.\n"); + + fi_version = FI_VERSION(1, 5); + + hints->domain_attr->mr_mode = FI_MR_BASIC; + hints->domain_attr->control_progress = FI_PROGRESS_AUTO; + hints->domain_attr->data_progress = FI_PROGRESS_AUTO; } + ifname = get_preferred_domain(Params); + forkunsafe = getenv("FI_FORK_UNSAFE"); if (!forkunsafe) { @@ -1638,7 +2133,7 @@ static int RdmaGetPriority(CP_Services Svcs, void *CP_Stream, struct _SstParams } pthread_mutex_lock(&fabric_mutex); - fi_getinfo(FI_VERSION(1, 5), NULL, NULL, 0, hints, &info); + fi_getinfo(fi_version, NULL, NULL, 0, hints, &info); pthread_mutex_unlock(&fabric_mutex); fi_freeinfo(hints); @@ -1656,6 +2151,10 @@ static int RdmaGetPriority(CP_Services Svcs, void *CP_Stream, struct _SstParams prov_name = info->fabric_attr->prov_name; domain_name = info->domain_attr->name; + Svcs->verbose(CP_Stream, DPPerStepVerbose, + "[RdmaGetPriority] Seeing and evaluating fabric with " + "provider: '%s', domain: '%s'\n", + prov_name, domain_name); if (ifname && strcmp(ifname, domain_name) == 0) { Svcs->verbose(CP_Stream, DPPerStepVerbose, @@ -1666,7 +2165,7 @@ static int RdmaGetPriority(CP_Services Svcs, void *CP_Stream, struct _SstParams break; } if ((strstr(prov_name, "verbs") && info->src_addr) || strstr(prov_name, "gni") || - strstr(prov_name, "psm2")) + strstr(prov_name, "psm2") || strstr(prov_name, "cxi")) { Svcs->verbose(CP_Stream, DPPerStepVerbose, @@ -1733,7 +2232,12 @@ static void PushData(CP_Services Svcs, Rdma_WSR_Stream Stream, TimestepList Step { rc = fi_writedata(Fabric->signal, StepBuffer + Req->Offset, Req->BufferLen, Step->Desc, Data, Stream->ReaderAddr[RankReq->Rank], - (uint64_t)Req->Handle.Block + + /* + * If mr_virt_addr is zero, we need just the offset, + * otherwise we need the remote virtual address composed by + * base pointer + offset. + */ + Fabric->mr_virt_addr * (uint64_t)Req->Handle.Block + (BufferSlot * RankReq->PreloadBufferSize), RollBuffer->Offset, (void *)(Step->Timestep)); } while (rc == -EAGAIN); @@ -1822,15 +2326,17 @@ static void PostPreload(CP_Services Svcs, Rdma_RS_Stream Stream, long Timestep) PreloadBuffer->BufferLen = 2 * StepLog->BufferSize; PreloadBuffer->Handle.Block = malloc(PreloadBuffer->BufferLen); - fi_mr_reg(Fabric->domain, PreloadBuffer->Handle.Block, PreloadBuffer->BufferLen, - FI_REMOTE_WRITE, 0, 0, 0, &Stream->pbmr, Fabric->ctx); + sst_fi_mr_reg(Svcs, Stream->CP_Stream, Fabric->domain, PreloadBuffer->Handle.Block, + PreloadBuffer->BufferLen, FI_REMOTE_WRITE, 0, 0, 0, &Stream->pbmr, Fabric->ctx, + Fabric->signal, Fabric->info->domain_attr->mr_mode); PreloadKey = fi_mr_key(Stream->pbmr); SBSize = sizeof(*SendBuffer) * StepLog->WRanks; SendBuffer = malloc(SBSize); if (Fabric->local_mr_req) { - fi_mr_reg(Fabric->domain, SendBuffer, SBSize, FI_WRITE, 0, 0, 0, &sbmr, Fabric->ctx); + sst_fi_mr_reg(Svcs, Stream->CP_Stream, Fabric->domain, SendBuffer, SBSize, FI_WRITE, 0, 0, + 0, &sbmr, Fabric->ctx, Fabric->signal, Fabric->info->domain_attr->mr_mode); sbdesc = fi_mr_desc(sbmr); } @@ -1838,8 +2344,9 @@ static void PostPreload(CP_Services Svcs, Rdma_RS_Stream Stream, long Timestep) { RBLen = 2 * StepLog->Entries * DP_DATA_RECV_SIZE; Stream->RecvDataBuffer = malloc(RBLen); - fi_mr_reg(Fabric->domain, Stream->RecvDataBuffer, RBLen, FI_RECV, 0, 0, 0, &Stream->rbmr, - Fabric->ctx); + sst_fi_mr_reg(Svcs, Stream->CP_Stream, Fabric->domain, Stream->RecvDataBuffer, RBLen, + FI_RECV, 0, 0, 0, &Stream->rbmr, Fabric->ctx, Fabric->signal, + Fabric->info->domain_attr->mr_mode); Stream->rbdesc = fi_mr_desc(Stream->rbmr); RecvBuffer = (uint8_t *)Stream->RecvDataBuffer; for (i = 0; i < 2 * StepLog->Entries; i++) @@ -1862,9 +2369,10 @@ static void PostPreload(CP_Services Svcs, Rdma_RS_Stream Stream, long Timestep) if (RankLog->Entries > 0) { RankLog->Buffer = (void *)RawPLBuffer; - fi_mr_reg(Fabric->domain, RankLog->ReqLog, - (sizeof(struct _RdmaBuffer) * RankLog->Entries) + sizeof(uint64_t), - FI_REMOTE_READ, 0, 0, 0, &RankLog->preqbmr, Fabric->ctx); + sst_fi_mr_reg(Svcs, Stream->CP_Stream, Fabric->domain, RankLog->ReqLog, + (sizeof(struct _RdmaBuffer) * RankLog->Entries) + sizeof(uint64_t), + FI_REMOTE_READ, 0, 0, 0, &RankLog->preqbmr, Fabric->ctx, Fabric->signal, + Fabric->info->domain_attr->mr_mode); for (j = 0; j < RankLog->Entries; j++) { ReqLog = &RankLog->ReqLog[j]; @@ -1883,11 +2391,17 @@ static void PostPreload(CP_Services Svcs, Rdma_RS_Stream Stream, long Timestep) SendBuffer[WRidx].Offset = (uint64_t)PreloadKey; SendBuffer[WRidx].Handle.Block = (void *)RankLog->ReqLog; SendBuffer[WRidx].Handle.Key = fi_mr_key(RankLog->preqbmr); - RollDest = - (uint64_t)Stream->WriterRoll[i].Block + (sizeof(struct _RdmaBuffer) * Stream->Rank); - fi_write(Fabric->signal, &SendBuffer[WRidx], sizeof(struct _RdmaBuffer), sbdesc, - Stream->WriterAddr[i], RollDest, Stream->WriterRoll[i].Key, - &SendBuffer[WRidx]); + /* + * If mr_virt_addr is zero, we need just the offset, + * otherwise we need the remote virtual address composed by + * base pointer + offset. + */ + RollDest = Fabric->mr_virt_addr * (uint64_t)Stream->WriterRoll[i].Block + + (sizeof(struct _RdmaBuffer) * Stream->Rank); + guard_fi_return((int)fi_write(Fabric->signal, &SendBuffer[WRidx], + sizeof(struct _RdmaBuffer), sbdesc, Stream->WriterAddr[i], + RollDest, Stream->WriterRoll[i].Key, &SendBuffer[WRidx]), + Svcs, Stream->CP_Stream, "[PostPreload] fi_write failed with:"); RankLog->PreloadHandles = malloc(sizeof(void *) * 2); RankLog->PreloadHandles[0] = calloc(sizeof(struct _RdmaCompletionHandle), RankLog->Entries); @@ -1899,7 +2413,19 @@ static void PostPreload(CP_Services Svcs, Rdma_RS_Stream Stream, long Timestep) while (WRidx > 0) { - fi_cq_sread(Fabric->cq_signal, (void *)(&CQEntry), 1, NULL, -1); + ssize_t rc = fi_cq_sread(Fabric->cq_signal, (void *)(&CQEntry), 1, NULL, -1); + if (rc < 1) + { + struct fi_cq_err_entry error; + fi_cq_readerr(Fabric->cq_signal, &error, 0); + Svcs->verbose( + Stream->CP_Stream, DPCriticalVerbose, + "[PostPreload] failure while waiting for completions " + "(%d (%s - %s)).\n", + rc, fi_strerror(error.err), + fi_cq_strerror(Fabric->cq_signal, error.err, error.err_data, NULL, error.len)); + return; + } CQBuffer = CQEntry.op_context; if (CQBuffer >= SendBuffer && CQBuffer < (SendBuffer + StepLog->WRanks)) { @@ -2000,24 +2526,45 @@ static void PullSelection(CP_Services Svcs, Rdma_WSR_Stream Stream) ReqBuffer.Handle.Block = ReadBuffer = malloc(ReqBuffer.BufferLen); if (Fabric->local_mr_req) { - fi_mr_reg(Fabric->domain, ReqBuffer.Handle.Block, ReqBuffer.BufferLen, FI_READ, 0, 0, 0, - &rrmr, Fabric->ctx); + sst_fi_mr_reg(Svcs, WS_Stream->CP_Stream, Fabric->domain, ReqBuffer.Handle.Block, + ReqBuffer.BufferLen, FI_READ, 0, 0, 0, &rrmr, Fabric->ctx, Fabric->signal, + Fabric->info->domain_attr->mr_mode); rrdesc = fi_mr_desc(rrmr); } for (RankReq = Stream->PreloadReq; RankReq; RankReq = RankReq->next) { RankReq->ReqLog = (RdmaBuffer)ReadBuffer; - fi_read(Fabric->signal, RankReq->ReqLog, RankReq->BufferSize, rrdesc, - Stream->ReaderAddr[RankReq->Rank], (uint64_t)ReaderRoll[RankReq->Rank].Handle.Block, - ReaderRoll[RankReq->Rank].Handle.Key, RankReq); + guard_fi_return( + (int)fi_read(Fabric->signal, RankReq->ReqLog, RankReq->BufferSize, rrdesc, + Stream->ReaderAddr[RankReq->Rank], + /* + * If mr_virt_addr is 0, then this is a simple + * null-pointer, indicating no offset. Otherwise, we + * need the remote virtual memory read address. + */ + Fabric->mr_virt_addr * (uint64_t)ReaderRoll[RankReq->Rank].Handle.Block, + ReaderRoll[RankReq->Rank].Handle.Key, RankReq), + Svcs, WS_Stream->CP_Stream, "[PullSelection] fi_read() failed with:"); ReadBuffer += RankReq->BufferSize; } RankReq = Stream->PreloadReq; while (RankReq) { - fi_cq_sread(Fabric->cq_signal, (void *)(&CQEntry), 1, NULL, -1); + ssize_t rc = fi_cq_sread(Fabric->cq_signal, (void *)(&CQEntry), 1, NULL, -1); + if (rc < 1) + { + struct fi_cq_err_entry error; + fi_cq_readerr(Fabric->cq_signal, &error, 0); + Svcs->verbose( + WS_Stream->CP_Stream, DPCriticalVerbose, + "[PullSelection] failure while waiting for completions " + "(%d (%s - %s)).\n", + rc, fi_strerror(error.err), + fi_cq_strerror(Fabric->cq_signal, error.err, error.err_data, NULL, error.len)); + return; + } CQRankReq = CQEntry.op_context; if (CQEntry.flags & FI_READ) { @@ -2049,7 +2596,19 @@ static void CompletePush(CP_Services Svcs, Rdma_WSR_Stream Stream, TimestepList while (Step->OutstandingWrites > 0) { - fi_cq_sread(Fabric->cq_signal, (void *)(&CQEntry), 1, NULL, -1); + ssize_t rc = fi_cq_sread(Fabric->cq_signal, (void *)(&CQEntry), 1, NULL, -1); + if (rc < 1) + { + struct fi_cq_err_entry error; + fi_cq_readerr(Fabric->cq_signal, &error, 0); + Svcs->verbose( + WS_Stream->CP_Stream, DPCriticalVerbose, + "[CompletePush] failure while waiting for completions " + "(%d (%s - %s)).\n", + rc, fi_strerror(error.err), + fi_cq_strerror(Fabric->cq_signal, error.err, error.err_data, NULL, error.len)); + return; + } if (CQEntry.flags & FI_WRITE) { CQTimestep = (long)CQEntry.op_context; From f843ae9747985a52c1d06e8d6709b60eb052f635 Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Wed, 6 Dec 2023 15:43:43 -0700 Subject: [PATCH 17/18] Ignore tests with defects for now --- CTestConfig.cmake | 175 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) diff --git a/CTestConfig.cmake b/CTestConfig.cmake index 67ef8c43ee..49d6e3826c 100644 --- a/CTestConfig.cmake +++ b/CTestConfig.cmake @@ -11,3 +11,178 @@ set(CTEST_DROP_SITE "open.cdash.org") set(CTEST_DROP_LOCATION "/submit.php?project=ADIOS") set(CTEST_DROP_SITE_CDASH TRUE) set(MEMORYCHECK_SUPPRESSIONS_FILE ${CMAKE_SOURCE_DIR}/scripts/dashboard/nightly/valgrind-suppressions.txt) + +# Ignore tests that are currently failing, remove tests here as they are fixed +list(APPEND CTEST_CUSTOM_MEMCHECK_IGNORE + Bindings.C.ADIOS2_C_API.ADIOS2BPWriteTypes.Serial + Bindings.C.BPWriteReadMultiblockCC.ZeroSizeBlocks.Serial + Engine.BP.*/BPAppendAfterStepsP.Test/*.BP5.Serial + Engine.BP.*/BPChangingShapeWithinStep.MultiBlock/*.BP5.Serial + Engine.BP.*/BPParameterSelectStepsP.Read/*.BP5.Serial + Engine.BP.*/BPReadMultithreadedTestP.ReadFile/*.BP5.Serial + Engine.BP.*/BPStepsFileGlobalArrayParameters.EveryOtherStep/*.BP5.Serial + Engine.BP.*/BPStepsFileGlobalArrayReaders.EveryStep/*.BP5.Serial + Engine.BP.*/BPStepsFileGlobalArrayReaders.NewVarPerStep/*.BP5.Serial + Engine.BP.*/BPStepsFileLocalArrayParameters.EveryOtherStep/*.BP5.Serial + Engine.BP.*/BPStepsFileLocalArrayReaders.EveryStep/*.BP5.Serial + Engine.BP.*/BPStepsFileLocalArrayReaders.NewVarPerStep/*.BP5.Serial + Engine.BP.BPChangingShape.BPWriteReadShape2D.BP5.Serial + Engine.BP.BPLargeMetadata.ManyLongStrings.BP5.Serial + Engine.BP.BPWriteAppendReadTestADIOS2.ADIOS2BPWriteAppendRead2D2x4.BP5.Serial + Engine.BP.BPWriteAppendReadTestADIOS2.ADIOS2BPWriteAppendReadAggregate.BP5.Serial + Engine.BP.BPWriteAppendReadTestADIOS2.ADIOS2BPWriteAppendReadVaryingAggregation.BP5.Serial + Engine.BP.BPWriteMultiblockReadTest.ADIOS2BPWriteMultiblockRead1D8.BP5.Serial + Engine.BP.BPWriteMultiblockReadTest.ADIOS2BPWriteMultiblockRead2D2x4.BP5.Serial + Engine.BP.BPWriteMultiblockReadTest.ADIOS2BPWriteMultiblockRead2D4x2.BP5.Serial + Engine.BP.BPWriteMultiblockReadTest.ADIOS2BPWriteRead1D8ZeroBlock.BP5.Serial + Engine.BP.BPWriteReadAsStreamTestADIOS2.ReaderWriterDefineVariable.BP5.Serial + Engine.BP.BPWriteReadAttributes.BPWriteReadSingleTypesVar.BP5.Serial + Engine.BP.BPWriteReadAttributes.WriteReadArrayTypes.BP5.Serial + Engine.BP.BPWriteReadAttributes.WriteReadArrayTypesVar.BP5.Serial + Engine.BP.BPWriteReadAttributes.WriteReadSingleTypes.BP5.Serial + Engine.BP.BPWriteReadAttributes.WriteReadStreamVarp.BP5.Serial + Engine.BP.BPWriteReadAttributeTestMultirank.ADIOS2BPWriteReadArrayTypes.BP5.Serial + Engine.BP.BPWriteReadBlockInfo.BPWriteReadBlockInfo1D8_C.BP3.Serial + Engine.BP.BPWriteReadBlockInfo.BPWriteReadBlockInfo1D8_C.BP4.Serial + Engine.BP.BPWriteReadBlockInfo.BPWriteReadBlockInfo1D8_C.BP5.Serial + Engine.BP.BPWriteReadBlockInfo.BPWriteReadBlockInfo1D8.BP5.Serial + Engine.BP.BPWriteReadBlockInfo.BPWriteReadBlockInfo2D2x4.BP5.Serial + Engine.BP.BPWriteReadLocalVariables.ADIOS2BPWriteReadLocal1DAllSteps.BP5.Serial + Engine.BP.BPWriteReadLocalVariables.ADIOS2BPWriteReadLocal1DBlockInfo.BP5.Serial + Engine.BP.BPWriteReadLocalVariables.ADIOS2BPWriteReadLocal2DChangeCount.BP3.Serial + Engine.BP.BPWriteReadLocalVariables.ADIOS2BPWriteReadLocal2DChangeCount.BP4.Serial + Engine.BP.BPWriteReadLocalVariables.ADIOS2BPWriteReadLocal2DChangeCount.BP5.Serial + Engine.BP.BPWriteReadLocalVariablesSel.BPWriteReadLocal1DAllStepsSel.BP5.Serial + Engine.BP.BPWriteReadLocalVariablesSelHighLevel.BPWriteReadLocal1DAllStepsSel.BP5.Serial + Engine.BP.BPWriteReadMultiblockTest.ADIOS2BPWriteReadMultiblock1D8.BP5.Serial + Engine.BP.BPWriteReadMultiblockTest.ADIOS2BPWriteReadMultiblock2D2x4.BP5.Serial + Engine.BP.BPWriteReadMultiblockTest.ADIOS2BPWriteReadMultiblock2D4x2.BP5.Serial + Engine.BP.BPWriteReadMultiblockTest.MultiblockNullBlocks.BP5.Serial + Engine.BP.BPWriteReadMultiblockTest.MultiblockPerformDataWrite.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead10D2x2.Async.BP5.EWS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead10D2x2.Async.BP5.EWS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead10D2x2.Async.BP5.TLS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead10D2x2.Async.BP5.TLS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead10D2x2.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead1D8.Async.BP5.EWS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead1D8.Async.BP5.EWS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead1D8.Async.BP5.TLS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead1D8.Async.BP5.TLS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead1D8.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D2x4.Async.BP5.EWS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D2x4.Async.BP5.EWS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D2x4.Async.BP5.TLS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D2x4.Async.BP5.TLS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D2x4.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2_MultiStepsOverflow.Async.BP5.EWS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2_MultiStepsOverflow.Async.BP5.EWS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2_MultiStepsOverflow.Async.BP5.TLS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2_MultiStepsOverflow.Async.BP5.TLS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2_MultiStepsOverflow.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2_ReadMultiSteps.Async.BP5.EWS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2_ReadMultiSteps.Async.BP5.EWS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2_ReadMultiSteps.Async.BP5.TLS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2_ReadMultiSteps.Async.BP5.TLS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2_ReadMultiSteps.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2.Async.BP5.EWS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2.Async.BP5.EWS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2.Async.BP5.TLS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2.Async.BP5.TLS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2.GetDeferredInClose.Async.BP5.EWS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.GetDeferredInClose.Async.BP5.EWS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.GetDeferredInClose.Async.BP5.TLS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.GetDeferredInClose.Async.BP5.TLS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.GetDeferredInClose.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2.ReadStartCount.Async.BP5.EWS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.ReadStartCount.Async.BP5.EWS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.ReadStartCount.Async.BP5.TLS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.ReadStartCount.Async.BP5.TLS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.ReadStartCount.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2fstream.ADIOS2BPWriteRead1D8.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2fstream.ADIOS2BPWriteRead2D2x4.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2fstream.ADIOS2BPWriteRead2D4x2_MultiStepsOverflow.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2fstream.ADIOS2BPWriteRead2D4x2_ReadMultiSteps.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2fstream.ADIOS2BPWriteRead2D4x2.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2stdio.ADIOS2BPWriteRead1D8.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2stdio.ADIOS2BPWriteRead2D2x4.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2stdio.ADIOS2BPWriteRead2D4x2_MultiStepsOverflow.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2stdio.ADIOS2BPWriteRead2D4x2_ReadMultiSteps.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2stdio.ADIOS2BPWriteRead2D4x2.BP5.Serial + Engine.BP.BPWriteReadVector.ADIOS2BPWriteRead1D8.BP5.Serial + Engine.BP.BPWriteReadVector.ADIOS2BPWriteRead2D2x4.BP5.Serial + Engine.BP.BPWriteReadVector.ADIOS2BPWriteRead2D4x2.BP5.Serial + Engine.BP.BPWriteReadVector.ADIOS2BPWriteReadVector2D4x2_MultiSteps.BP5.Serial + Engine.BPEngineTest.ZfpComplex.Serial + Engine.SST.SstWriteFails.InvalidBeginStep.Serial + Engine.SST.SstWriteFails.InvalidPut.Serial + Engine.Staging.TestThreads.Basic.SST.BP.Serial + Engine.Staging.TestThreads.Basic.SST.FFS.Serial + Interface.ADIOS2_CXX11_API_Selection.MemorySelectionComplex.BPfile.Serial + Interface.ADIOS2_CXX11_API_Selection.MemorySelectionReadStart.BPfile.Serial + Remote.BPWriteMemorySelectionRead.FileRemote + Remote.BPWriteMemorySelectionRead.GetRemote + Remote.BPWriteReadADIOS2stdio.GetRemote + remoteServerCleanup + remoteServerSetup + Staging.1x1.Attrs.BP5 + Staging.1x1.Attrs.CommMin.BP.SST + Staging.1x1.Attrs.CommMin.BP5.SST + Staging.1x1.CommMin.BP.SST + Staging.1x1.CommMin.BP5.SST + Staging.1x1.ForcePreload.CommMin.BP.SST + Staging.1x1.ForcePreload.CommMin.BP5.SST + Staging.1x1.Local2.CommMin.BP.SST + Staging.1x1.Local2.CommMin.BP5.SST + Staging.1x1.LocalMultiblock.CommMin.BP.SST + Staging.1x1.LocalMultiblock.CommMin.BP5.SST + Staging.1x1.ModAttrs.BP5 + Staging.1x1.ModAttrs.CommMin.BP.SST + Staging.1x1.ModAttrs.CommMin.BP5.SST + Staging.1x1.NoPreload.CommMin.BP.SST + Staging.1x1.NoPreload.CommMin.BP5.SST + Staging.1x1.SstRUDP.CommMin.BP.SST + Staging.1x1.SstRUDP.CommMin.BP5.SST + Staging.1x1Joined.BP5 + Staging.1x1Joined.CommMin.BP5.SST + Staging.1x1LockGeometry.CommMin.BP.SST + Staging.1x1LockGeometry.CommMin.BP5.SST + Staging.1x1Struct.BP5 + Staging.1x1Struct.CommMin.BP5.SST + Staging.1x1VarDestruction.CommMin.BP.SST + Staging.1x1VarDestruction.CommMin.BP5.SST + Staging.AllToAllDistribution.1x1x3.CommMin.BP.SST + Staging.AllToAllDistribution.1x1x3.CommMin.BP5.SST + Staging.DiscardWriter.1x1.CommMin.BP.SST + Staging.DiscardWriter.1x1.CommMin.BP5.SST + Staging.LatestReader.1x1.CommMin.BP.SST + Staging.LatestReader.1x1.CommMin.BP5.SST + Staging.LatestReaderHold.1x1.CommMin.BP.SST + Staging.LatestReaderHold.1x1.CommMin.BP5.SST + Staging.OnDemandSingle.1x1.CommMin.BP.SST + Staging.OnDemandSingle.1x1.CommMin.BP5.SST + Staging.RoundRobinDistribution.1x1x3.CommMin.BP.SST + Staging.RoundRobinDistribution.1x1x3.CommMin.BP5.SST + Staging.TimeoutReader.1x1.CommMin.BP.SST + Staging.TimeoutReader.1x1.CommMin.BP5.SST + Staging.WriteMemorySelectionRead.1x1.CommMin.BP.SST + Staging.WriteMemorySelectionRead.1x1.CommMin.BP5.SST + Staging.ZFPCompression.1x1.CommMin.BP.SST + Staging.ZFPCompression.1x1.CommMin.BP5.SST + Staging.ZFPCompression.3x5.CommMin.BP.SST + Staging.ZFPCompression.3x5.CommMin.BP5.SST + Utils.ChangingShape.AlternatingStepsAndChangingShapeVar.Dump + Utils.ChangingShape.AlternatingStepsVarSelection.Dump + Utils.ChangingShape.ChangingShapeVarOneStep.Dump + Utils.ChangingShape.Dump + Utils.ChangingShape.FixedShapeVarTooManySteps.Dump + Utils.ChangingShape.Screen + Utils.CWriter + Utils.CWriter.Bpls.Al.Dump + Utils.CWriter.Bpls.h.Dump + Utils.CWriter.Bpls.la.Dump + Utils.CWriter.Bpls.la.Screen + Utils.CWriter.Bpls.ldDav.Dump + Utils.CWriter.Bpls.ldDavvv.Dump + Utils.CWriter.Bpls.ldvarI16.Dump +) From 5363c5d7efd551367d933e50575b5783516db1d3 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Wed, 8 Nov 2023 09:57:03 -0500 Subject: [PATCH 18/18] Update readme for heat transfer example with new location and build instructions --- examples/simulations/heatTransfer/ReadMe.md | 26 +++++++++++++-------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/examples/simulations/heatTransfer/ReadMe.md b/examples/simulations/heatTransfer/ReadMe.md index 8693d00265..b1a45a24e7 100644 --- a/examples/simulations/heatTransfer/ReadMe.md +++ b/examples/simulations/heatTransfer/ReadMe.md @@ -22,6 +22,13 @@ an application to the ADIOS2 library for its IO. #### Example +##### 1. Build the example +$ mkdir build +$ cd build +$ cmake -DCMAKE_PREFIX_PATH= -DCMAKE_INSTALL_PREFIX=install .. +$ make -j 8 +$ cd .. + ##### 1. Produce an output ``` @@ -44,24 +51,23 @@ The adios1, ph5 and hdf5 versions of the example do not use XML config files, so argument. ``` -$ mpirun -np 12 ./bin/adios2_simulations_heatTransferWrite ../examples/simulations/heatTransfer/heat_file.xml heat.bp 4 -3 5 10 10 10 -$ mpirun -np 12 ./bin/adios2_simulations_heatTransferWrite ../examples/simulations/heatTransfer/heat_file.xml heat.h5 4 -3 5 10 10 10 +$ mpirun -np 12 ./build/write/adios2_simulations_heatTransferWrite heat_file.xml heat.bp 4 3 5 10 10 10 +$ mpirun -np 12 ./build/write/adios2_simulations_heatTransferWrite heat_hdf5.xml heat.h5 4 3 5 10 10 10 ``` ##### 2. Read the output step-by-step and print data into text files (data. per reader process) ``` -Reader Usage: heatRead config input N M - config: XML config file to use - input: name of input data file/stream - N: number of processes in X dimension - M: number of processes in Y dimension +Reader Usage: heatRead config input output N M + config: XML config file to use + input: name of input data file/stream + output: name of output data file/stream + N: number of processes in X dimension + M: number of processes in Y dimension ``` ``` -$ mpirun -np 2 ./bin/heatTransfer_read ../examples/simulations/heatTransfer/heat_file.xml heat 2 1 +$ mpirun -np 2 ./build/read/adios2_simulations_heatTransferRead heat_file.xml heat.bp read.bp 2 1 ``` ##### Notes: