From b9347e58eb79d604635e7aa9ae66e35d5cda99ae Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Tue, 2 Jun 2020 19:54:37 -0500 Subject: [PATCH 01/30] Update CDash URL --- CTestConfig.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CTestConfig.cmake b/CTestConfig.cmake index 138d4683..4769bb0d 100644 --- a/CTestConfig.cmake +++ b/CTestConfig.cmake @@ -8,8 +8,8 @@ set(CTEST_PROJECT_NAME "MERCURY") set(CTEST_NIGHTLY_START_TIME "00:00:00 CST") -set(CTEST_DROP_METHOD "http") -set(CTEST_DROP_SITE "149.28.123.102/CDash") +set(CTEST_DROP_METHOD "https") +set(CTEST_DROP_SITE "mercury-cdash.hdfgroup.org") set(CTEST_DROP_LOCATION "/submit.php?project=Mercury") set(CTEST_DROP_SITE_CDASH TRUE) From 0e16255f47ae4e892d8f0cf7f0e2d702cf636423 Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Thu, 14 May 2020 17:21:34 -0500 Subject: [PATCH 02/30] Add clang format file --- .clang-format | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .clang-format diff --git a/.clang-format b/.clang-format new file mode 100644 index 00000000..2183349b --- /dev/null +++ b/.clang-format @@ -0,0 +1,18 @@ +--- +BasedOnStyle: LLVM +UseTab: Never +IndentWidth: 4 +TabWidth: 4 +BreakBeforeBraces: Linux +AllowShortIfStatementsOnASingleLine: false +IndentCaseLabels: true +IndentGotoLabels: false +IndentPPDirectives: AfterHash +ColumnLimit: 80 +AccessModifierOffset: -4 +AlignAfterOpenBracket: DontAlign +AlignConsecutiveMacros: true +AlwaysBreakAfterReturnType: All +SpaceAfterCStyleCast: true +ForEachMacros: ['HG_QUEUE_FOREACH', 'HG_LIST_FOREACH'] +... From 036c0e85e3ac6716b966f2c1c386c93225396035 Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Thu, 14 May 2020 17:24:45 -0500 Subject: [PATCH 03/30] Add gitignore file --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..5acb669b --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +build +.vscode From 550423af6a76e2b87bccd8909cefef7de461b090 Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Thu, 14 May 2020 18:00:16 -0500 Subject: [PATCH 04/30] CMake: add support for Asan/Tsan builds --- CMake/CheckAsan.cmake | 37 +++++++++++++++++++++++++++++++++++ CMake/CheckTsan.cmake | 37 +++++++++++++++++++++++++++++++++++ CMakeLists.txt | 16 ++++++++++----- Testing/driver/CMakeLists.txt | 3 +++ src/CMakeLists.txt | 16 +++++++-------- src/mchecksum | 2 +- 6 files changed, 97 insertions(+), 14 deletions(-) create mode 100644 CMake/CheckAsan.cmake create mode 100644 CMake/CheckTsan.cmake diff --git a/CMake/CheckAsan.cmake b/CMake/CheckAsan.cmake new file mode 100644 index 00000000..32f4b453 --- /dev/null +++ b/CMake/CheckAsan.cmake @@ -0,0 +1,37 @@ +set(ASAN_FLAG "-fsanitize=address") +set(ASAN_C_FLAGS "-O1 -g ${ASAN_FLAG} -fsanitize-address-use-after-scope -fno-omit-frame-pointer -fno-optimize-sibling-calls") +set(ASAN_CXX_FLAGS ${ASAN_C_FLAGS}) + +get_property(ASAN_LANGUAGES GLOBAL PROPERTY ENABLED_LANGUAGES) +foreach(lang ${ASAN_LANGUAGES}) + set(ASAN_${lang}_LANG_ENABLED 1) +endforeach() + +if(ASAN_C_LANG_ENABLED) + include(CheckCCompilerFlag) + set(CMAKE_REQUIRED_LINK_OPTIONS ${ASAN_FLAG}) + check_c_compiler_flag(${ASAN_FLAG} ASAN_C_FLAG_SUPPORTED) + if(NOT ASAN_C_FLAG_SUPPORTED) + message(STATUS "Asan flags are not supported by the C compiler.") + else() + if(NOT CMAKE_C_FLAGS_ASAN) + set(CMAKE_C_FLAGS_ASAN ${ASAN_C_FLAGS} CACHE STRING "Flags used by the C compiler during ASAN builds." FORCE) + endif() + endif() + unset(CMAKE_REQUIRED_LINK_OPTIONS) +endif() + +if(ASAN_CXX_LANG_ENABLED) + include(CheckCXXCompilerFlag) + set(CMAKE_REQUIRED_LINK_OPTIONS ${ASAN_FLAG}) + check_cxx_compiler_flag(${ASAN_FLAG} ASAN_CXX_FLAG_SUPPORTED) + if(NOT ASAN_CXX_FLAG_SUPPORTED) + message(STATUS "Asan flags are not supported by the CXX compiler.") + else() + if(NOT CMAKE_CXX_FLAGS_ASAN) + set(CMAKE_CXX_FLAGS_ASAN ${ASAN_CXX_FLAGS} CACHE STRING "Flags used by the CXX compiler during ASAN builds." FORCE) + endif() + endif() + unset(CMAKE_REQUIRED_LINK_OPTIONS) +endif() + diff --git a/CMake/CheckTsan.cmake b/CMake/CheckTsan.cmake new file mode 100644 index 00000000..9ebcb2e5 --- /dev/null +++ b/CMake/CheckTsan.cmake @@ -0,0 +1,37 @@ +set(TSAN_FLAG "-fsanitize=thread") +set(TSAN_C_FLAGS "-O1 -g ${TSAN_FLAG}") +set(TSAN_CXX_FLAGS ${TSAN_C_FLAGS}) + +get_property(TSAN_LANGUAGES GLOBAL PROPERTY ENABLED_LANGUAGES) +foreach(lang ${TSAN_LANGUAGES}) + set(TSAN_${lang}_LANG_ENABLED 1) +endforeach() + +if(TSAN_C_LANG_ENABLED) + include(CheckCCompilerFlag) + set(CMAKE_REQUIRED_LINK_OPTIONS ${TSAN_FLAG}) + check_c_compiler_flag(${TSAN_FLAG} TSAN_C_FLAG_SUPPORTED) + if(NOT TSAN_C_FLAG_SUPPORTED) + message(STATUS "Asan flags are not supported by the C compiler.") + else() + if(NOT CMAKE_C_FLAGS_TSAN) + set(CMAKE_C_FLAGS_TSAN ${TSAN_C_FLAGS} CACHE STRING "Flags used by the C compiler during TSAN builds." FORCE) + endif() + endif() + unset(CMAKE_REQUIRED_LINK_OPTIONS) +endif() + +if(TSAN_CXX_LANG_ENABLED) + include(CheckCXXCompilerFlag) + set(CMAKE_REQUIRED_LINK_OPTIONS ${TSAN_FLAG}) + check_cxx_compiler_flag(${TSAN_FLAG} TSAN_CXX_FLAG_SUPPORTED) + if(NOT TSAN_CXX_FLAG_SUPPORTED) + message(STATUS "Asan flags are not supported by the CXX compiler.") + else() + if(NOT CMAKE_CXX_FLAGS_TSAN) + set(CMAKE_CXX_FLAGS_TSAN ${TSAN_CXX_FLAGS} CACHE STRING "Flags used by the CXX compiler during TSAN builds." FORCE) + endif() + endif() + unset(CMAKE_REQUIRED_LINK_OPTIONS) +endif() + diff --git a/CMakeLists.txt b/CMakeLists.txt index 1f6fa8ee..399fe366 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -138,7 +138,7 @@ if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "Choose the type of build." FORCE) # Set the possible values of build type for cmake-gui set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" - "MinSizeRel" "RelWithDebInfo") + "MinSizeRel" "RelWithDebInfo" "Asan" "Tsan") endif() if(NOT CMAKE_C_FLAGS AND CMAKE_COMPILER_IS_GNUCC) @@ -146,6 +146,10 @@ if(NOT CMAKE_C_FLAGS AND CMAKE_COMPILER_IS_GNUCC) set(CMAKE_C_FLAGS "-Wall -Wextra -Winline -Wcast-qual -std=gnu99 -Wshadow" CACHE STRING "Flags used by the compiler during all build types." FORCE) endif() +# Detect Asan and Tsan compiler flags +include(CheckAsan) +include(CheckTsan) + #----------------------------------------------------------------------------- # Targets built within this project are exported at Install time for use # by other projects. @@ -212,10 +216,12 @@ function(mercury_set_lib_options libtarget libname libtype) set_target_properties(${libtarget} PROPERTIES - DEBUG_OUTPUT_NAME ${LIB_DEBUG_NAME} - RELEASE_OUTPUT_NAME ${LIB_RELEASE_NAME} - MINSIZEREL_OUTPUT_NAME ${LIB_RELEASE_NAME} - RELWITHDEBINFO_OUTPUT_NAME ${LIB_RELEASE_NAME} + OUTPUT_NAME_DEBUG ${LIB_DEBUG_NAME} + OUTPUT_NAME_RELEASE ${LIB_RELEASE_NAME} + OUTPUT_NAME_MINSIZEREL ${LIB_RELEASE_NAME} + OUTPUT_NAME_RELWITHDEBINFO ${LIB_RELEASE_NAME} + OUTPUT_NAME_ASAN ${LIB_DEBUG_NAME} + OUTPUT_NAME_TSAN ${LIB_DEBUG_NAME} VERSION ${LIB_VERSION} SOVERSION ${API_VERSION} ) diff --git a/Testing/driver/CMakeLists.txt b/Testing/driver/CMakeLists.txt index 3055e121..59a9e9fc 100644 --- a/Testing/driver/CMakeLists.txt +++ b/Testing/driver/CMakeLists.txt @@ -1,6 +1,9 @@ cmake_minimum_required(VERSION 2.8.12.2 FATAL_ERROR) project(MERCURY_TEST_DRIVER CXX) +include(CheckAsan) +include(CheckTsan) + set(KWSYS_NAMESPACE mercury_sys) set(KWSYS_USE_SystemTools 1) set(KWSYS_USE_Process 1) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f258f24c..d2f46784 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -327,12 +327,12 @@ install( # Export all exported targets to the build tree for use by parent project #----------------------------------------------------------------------------- if(NOT MERCURY_EXTERNALLY_CONFIGURED) -EXPORT ( - TARGETS - ${MERCURY_EXPORTED_LIBS} - FILE - ${MERCURY_EXPORTED_TARGETS}.cmake -) + export( + TARGETS + ${MERCURY_EXPORTED_LIBS} + FILE + ${MERCURY_EXPORTED_TARGETS}.cmake + ) endif() #------------------------------------------------------------------------------ @@ -355,9 +355,9 @@ if(NOT WIN32) # Mercury private library dependencies foreach(exported_lib ${MERCURY_EXPORTED_LIBS}) if(lower_cmake_build_type MATCHES "debug") - get_target_property(MERCURY_LIBRARY ${exported_lib} DEBUG_OUTPUT_NAME) + get_target_property(MERCURY_LIBRARY ${exported_lib} OUTPUT_NAME_DEBUG) else() - get_target_property(MERCURY_LIBRARY ${exported_lib} RELEASE_OUTPUT_NAME) + get_target_property(MERCURY_LIBRARY ${exported_lib} OUTPUT_NAME_RELEASE) endif() set(MERCURY_LIBRARIES "${MERCURY_LIBRARIES} -l${MERCURY_LIBRARY}") endforeach() diff --git a/src/mchecksum b/src/mchecksum index 3c76b32e..5b64750b 160000 --- a/src/mchecksum +++ b/src/mchecksum @@ -1 +1 @@ -Subproject commit 3c76b32e5f693f03f51123a793b2032b49f45b16 +Subproject commit 5b64750b315a6b758012314c0be7b1776154692f From d70f9443ca10e17d9c6725d257787d03431e8157 Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Fri, 5 Jun 2020 14:46:41 -0500 Subject: [PATCH 05/30] CMake: Correct usage of FindThreads to add -pthread compile option --- src/CMakeLists.txt | 8 +++++--- src/util/CMakeLists.txt | 27 +++++++++++++++------------ 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d2f46784..a2a6281f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -372,14 +372,16 @@ if(NOT WIN32) foreach(lib_dep ${MERCURY_EXT_LIB_DEPENDENCIES}) # get library name get_filename_component(lib_name ${lib_dep} NAME_WE) - if(lib_name MATCHES "^-l") + if(lib_name MATCHES "^-l" OR lib_name MATCHES "-pthread") # lib_name found is -lxxx - set(MERCURY_EXT_LIB_DEPENDENCIES_LIST ${MERCURY_EXT_LIB_DEPENDENCIES_LIST} ${lib_name}) + set(MERCURY_EXT_LIB_DEPENDENCIES_LIST + ${MERCURY_EXT_LIB_DEPENDENCIES_LIST} ${lib_name}) else() # lib_name is /path/to/lib so get library path and name get_filename_component(lib_path ${lib_dep} PATH) string(REGEX REPLACE "^lib" "" lib_name ${lib_name}) - set(MERCURY_EXT_LIB_DEPENDENCIES_LIST ${MERCURY_EXT_LIB_DEPENDENCIES_LIST} -L${lib_path} -l${lib_name}) + set(MERCURY_EXT_LIB_DEPENDENCIES_LIST + ${MERCURY_EXT_LIB_DEPENDENCIES_LIST} -L${lib_path} -l${lib_name}) endif() endforeach() if(MERCURY_EXT_LIB_DEPENDENCIES_LIST) diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 88746473..452ea180 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -20,7 +20,10 @@ include(CheckSymbolExists) include(CheckTypeSize) # Threads -include(FindThreads) +set(CMAKE_THREAD_PREFER_PTHREAD TRUE) +set(THREADS_PREFER_PTHREAD_FLAG TRUE) +find_package(Threads REQUIRED) + set(MERCURY_UTIL_EXT_LIB_DEPENDENCIES ${MERCURY_UTIL_EXT_LIB_DEPENDENCIES} ${CMAKE_THREAD_LIBS_INIT} @@ -36,7 +39,8 @@ if(CMAKE_USE_PTHREADS_INIT) check_type_size(PTHREAD_MUTEX_ADAPTIVE_NP HG_UTIL_HAS_PTHREAD_MUTEX_ADAPTIVE_NP) # Detect pthread_condattr_setclock - check_symbol_exists(pthread_condattr_setclock pthread.h HG_UTIL_HAS_PTHREAD_CONDATTR_SETCLOCK) + check_symbol_exists(pthread_condattr_setclock pthread.h + HG_UTIL_HAS_PTHREAD_CONDATTR_SETCLOCK) endif() # Rt @@ -91,16 +95,12 @@ if(NOT WIN32) if(MERCURY_USE_OPA) # Use OpenPA if stdatomic is not available find_package(OPA REQUIRED) - if(OPA_FOUND) - message(STATUS "OPA include directory: ${OPA_INCLUDE_DIRS}") - set(HG_UTIL_HAS_OPA_PRIMITIVES_H 1) - set(MERCURY_UTIL_EXT_INCLUDE_DEPENDENCIES - ${MERCURY_UTIL_EXT_INCLUDE_DEPENDENCIES} - ${OPA_INCLUDE_DIRS} - ) - else() - message(FATAL_ERROR "Could not find OPA.") - endif() + message(STATUS "OPA include directory: ${OPA_INCLUDE_DIRS}") + set(HG_UTIL_HAS_OPA_PRIMITIVES_H 1) + set(MERCURY_UTIL_EXT_INCLUDE_DEPENDENCIES + ${MERCURY_UTIL_EXT_INCLUDE_DEPENDENCIES} + ${OPA_INCLUDE_DIRS} + ) endif() endif() @@ -159,6 +159,9 @@ endforeach() # UTIL add_library(mercury_util ${MERCURY_UTIL_SRCS}) +if(THREADS_HAVE_PTHREAD_ARG) + target_compile_options(mercury_util PUBLIC "${CMAKE_THREAD_LIBS_INIT}") +endif() target_include_directories(mercury_util PUBLIC "$" $ From c96d9bd6ade9847dcc1c2cfbc3fa1e8576fd4aff Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Thu, 14 May 2020 18:06:36 -0500 Subject: [PATCH 06/30] HG Util: fix hg_poll_wait() to return events Update hg_test_poll after hg_poll_wait() changes Align mercury_util_error to new error macros Clean up event set/get wrappers No longer install mercury_util_error.h Clean up and fix formatting to follow clang format file Add support for colored log Check atomic_long width to match stdatomic type definition --- Testing/util/test_poll.c | 48 +- src/na/na_mpi.c | 1 + src/util/CMakeLists.txt | 16 +- src/util/mercury_atomic.h | 91 ++-- src/util/mercury_atomic_queue.c | 18 +- src/util/mercury_atomic_queue.h | 74 ++- src/util/mercury_event.c | 43 +- src/util/mercury_event.h | 130 +++--- src/util/mercury_hash_table.c | 696 +++++++++++++++------------- src/util/mercury_hash_table.h | 84 ++-- src/util/mercury_list.h | 120 ++--- src/util/mercury_log.c | 56 ++- src/util/mercury_log.h | 51 +- src/util/mercury_mem.c | 144 +++--- src/util/mercury_mem.h | 27 +- src/util/mercury_poll.c | 557 ++++++++++------------ src/util/mercury_poll.h | 46 +- src/util/mercury_queue.h | 104 +++-- src/util/mercury_request.c | 65 +-- src/util/mercury_request.h | 73 +-- src/util/mercury_thread.c | 116 ++--- src/util/mercury_thread.h | 65 ++- src/util/mercury_thread_condition.c | 19 +- src/util/mercury_thread_condition.h | 73 ++- src/util/mercury_thread_mutex.c | 20 +- src/util/mercury_thread_mutex.h | 38 +- src/util/mercury_thread_pool.c | 155 ++++--- src/util/mercury_thread_pool.h | 50 +- src/util/mercury_thread_rwlock.c | 14 +- src/util/mercury_thread_rwlock.h | 58 ++- src/util/mercury_thread_spin.c | 26 +- src/util/mercury_thread_spin.h | 50 +- src/util/mercury_time.h | 126 +++-- src/util/mercury_util_config.h.in | 180 +++---- src/util/mercury_util_error.c | 20 + src/util/mercury_util_error.h | 94 +++- 36 files changed, 1806 insertions(+), 1742 deletions(-) create mode 100644 src/util/mercury_util_error.c diff --git a/Testing/util/test_poll.c b/Testing/util/test_poll.c index 2dfa72c1..878cbc5c 100644 --- a/Testing/util/test_poll.c +++ b/Testing/util/test_poll.c @@ -11,13 +11,13 @@ struct hg_test_poll_cb_args { }; static int -poll_cb(void *arg, int error, hg_util_bool_t *progressed) +poll_cb(void *arg, int error, struct hg_poll_event *event) { struct hg_test_poll_cb_args *poll_cb_args = (struct hg_test_poll_cb_args *) arg; (void) error; - hg_event_get(poll_cb_args->event_fd, progressed); + hg_event_get(poll_cb_args->event_fd, &event->progressed); return HG_UTIL_SUCCESS; } @@ -27,9 +27,9 @@ main(void) { struct hg_test_poll_cb_args poll_cb_args; hg_poll_set_t *poll_set; - hg_util_bool_t progressed; - int event_fd; - int ret = EXIT_SUCCESS; + struct hg_poll_event events[1]; + unsigned int nevents = 0; + int event_fd, ret = EXIT_SUCCESS; poll_set = hg_poll_create(); event_fd = hg_event_create(); @@ -43,43 +43,57 @@ main(void) hg_event_set(event_fd); /* Wait with timeout 0 */ - hg_poll_wait(poll_set, 0, &progressed); - if (!progressed) { + hg_poll_wait(poll_set, 0, 1, events, &nevents); + if ((nevents != 1) || !events[0].progressed) { /* We expect success */ - fprintf(stderr, "Error: did not progress correctly\n"); + fprintf(stderr, "Error: should have progressed\n"); ret = EXIT_FAILURE; + goto done; } /* Reset progressed */ - progressed = HG_UTIL_FALSE; + nevents = 0; + events[0].progressed = HG_UTIL_FALSE; /* Wait with timeout 0 */ - hg_poll_wait(poll_set, 0, &progressed); - if (progressed) { + hg_poll_wait(poll_set, 0, 1, events, &nevents); + if ((nevents != 1) || events[0].progressed) { /* We do not expect success */ - fprintf(stderr, "Error: did not progress correctly\n"); + fprintf(stderr, "Error: should not have progressed\n"); ret = EXIT_FAILURE; + goto done; } + /* Reset progressed */ + nevents = 0; + events[0].progressed = HG_UTIL_FALSE; + /* Wait with timeout */ - hg_poll_wait(poll_set, 100, &progressed); - if (progressed) { + hg_poll_wait(poll_set, 100, 1, events, &nevents); + if (nevents || events[0].progressed) { /* We do not expect success */ - fprintf(stderr, "Error: did not progress correctly\n"); + fprintf(stderr, "Error: should not have progressed\n"); ret = EXIT_FAILURE; + goto done; } /* Set event */ hg_event_set(event_fd); + /* Reset progressed */ + nevents = 0; + events[0].progressed = HG_UTIL_FALSE; + /* Wait with timeout */ - hg_poll_wait(poll_set, 1000, &progressed); - if (!progressed) { + hg_poll_wait(poll_set, 1000, 1, events, &nevents); + if (!nevents || !events[0].progressed) { /* We expect success */ fprintf(stderr, "Error: did not progress correctly\n"); ret = EXIT_FAILURE; + goto done; } +done: hg_poll_remove(poll_set, event_fd); hg_poll_destroy(poll_set); hg_event_destroy(event_fd); diff --git a/src/na/na_mpi.c b/src/na/na_mpi.c index e63b8c85..bc6acda7 100644 --- a/src/na/na_mpi.c +++ b/src/na/na_mpi.c @@ -13,6 +13,7 @@ #include "mercury_list.h" #include "mercury_time.h" +#include "mercury_thread.h" #include #include diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 452ea180..a77881f2 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -41,6 +41,9 @@ if(CMAKE_USE_PTHREADS_INIT) # Detect pthread_condattr_setclock check_symbol_exists(pthread_condattr_setclock pthread.h HG_UTIL_HAS_PTHREAD_CONDATTR_SETCLOCK) + + unset(CMAKE_EXTRA_INCLUDE_FILES) + unset(CMAKE_REQUIRED_LIBRARIES) endif() # Rt @@ -85,6 +88,10 @@ check_include_files("sys/event.h" HG_UTIL_HAS_SYSEVENT_H) if(NOT WIN32) # Detect stdatomic check_include_files("stdatomic.h" HG_UTIL_HAS_STDATOMIC_H) + # Detect size of atomic_long + set(CMAKE_EXTRA_INCLUDE_FILES stdatomic.h) + check_type_size(atomic_long HG_UTIL_ATOMIC_LONG_WIDTH) + unset(CMAKE_EXTRA_INCLUDE_FILES) # OpenPA option(MERCURY_USE_OPA "Use OpenPA for atomics." OFF) # Force use of OPA if is not found @@ -104,6 +111,13 @@ if(NOT WIN32) endif() endif() +# Colored output +option(MERCURY_ENABLE_LOG_COLOR "Use colored output for log." OFF) +if(MERCURY_ENABLE_LOG_COLOR) + set(HG_UTIL_HAS_LOG_COLOR 1) +endif() +mark_as_advanced(MERCURY_ENABLE_LOG_COLOR) + #------------------------------------------------------------------------------ # Configure module header files #------------------------------------------------------------------------------ @@ -144,6 +158,7 @@ set(MERCURY_UTIL_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/mercury_thread_pool.c ${CMAKE_CURRENT_SOURCE_DIR}/mercury_thread_rwlock.c ${CMAKE_CURRENT_SOURCE_DIR}/mercury_thread_spin.c + ${CMAKE_CURRENT_SOURCE_DIR}/mercury_util_error.c ) #---------------------------------------------------------------------------- @@ -211,7 +226,6 @@ set(MERCURY_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/mercury_thread_rwlock.h ${CMAKE_CURRENT_SOURCE_DIR}/mercury_thread_spin.h ${CMAKE_CURRENT_SOURCE_DIR}/mercury_time.h - ${CMAKE_CURRENT_SOURCE_DIR}/mercury_util_error.h ) #----------------------------------------------------------------------------- diff --git a/src/util/mercury_atomic.h b/src/util/mercury_atomic.h index 6ece1d7d..7a684b98 100644 --- a/src/util/mercury_atomic.h +++ b/src/util/mercury_atomic.h @@ -14,27 +14,45 @@ #include "mercury_util_config.h" #if defined(_WIN32) -# include -typedef struct { volatile LONG value; } hg_atomic_int32_t; -typedef struct { volatile LONGLONG value; } hg_atomic_int64_t; -# define HG_ATOMIC_VAR_INIT(x) { (x) } +# include +typedef struct { + volatile LONG value; +} hg_atomic_int32_t; +typedef struct { + volatile LONGLONG value; +} hg_atomic_int64_t; +# define HG_ATOMIC_VAR_INIT(x) \ + { \ + (x) \ + } #elif defined(HG_UTIL_HAS_OPA_PRIMITIVES_H) -# include +# include typedef OPA_int_t hg_atomic_int32_t; typedef OPA_ptr_t hg_atomic_int64_t; /* OPA has only limited 64-bit support */ -# define HG_ATOMIC_VAR_INIT(x) OPA_PTR_T_INITIALIZER(x) +# define HG_ATOMIC_VAR_INIT(x) OPA_PTR_T_INITIALIZER(x) #elif defined(HG_UTIL_HAS_STDATOMIC_H) -# include +# include typedef atomic_int hg_atomic_int32_t; +# if HG_UTIL_ATOMIC_LONG_WIDTH == 8 +typedef atomic_long hg_atomic_int64_t; +# else typedef atomic_llong hg_atomic_int64_t; -# define HG_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x) +# endif +# define HG_ATOMIC_VAR_INIT(x) ATOMIC_VAR_INIT(x) #elif defined(__APPLE__) -# include -typedef struct { volatile hg_util_int32_t value; } hg_atomic_int32_t; -typedef struct { volatile hg_util_int64_t value; } hg_atomic_int64_t; -# define HG_ATOMIC_VAR_INIT(x) { (x) } +# include +typedef struct { + volatile hg_util_int32_t value; +} hg_atomic_int32_t; +typedef struct { + volatile hg_util_int64_t value; +} hg_atomic_int64_t; +# define HG_ATOMIC_VAR_INIT(x) \ + { \ + (x) \ + } #else -# error "Not supported on this platform." +# error "Not supported on this platform." #endif #ifdef __cplusplus @@ -262,7 +280,7 @@ hg_atomic_set32(hg_atomic_int32_t *ptr, hg_util_int32_t value) #elif defined(__APPLE__) ptr->value = value; #else - #error "Not supported on this platform." +# error "Not supported on this platform." #endif } @@ -281,7 +299,7 @@ hg_atomic_get32(hg_atomic_int32_t *ptr) #elif defined(__APPLE__) ret = ptr->value; #else - #error "Not supported on this platform." +# error "Not supported on this platform." #endif return ret; @@ -302,7 +320,7 @@ hg_atomic_incr32(hg_atomic_int32_t *ptr) #elif defined(__APPLE__) ret = OSAtomicIncrement32(&ptr->value); #else - #error "Not supported on this platform." +# error "Not supported on this platform." #endif return ret; @@ -323,7 +341,7 @@ hg_atomic_decr32(hg_atomic_int32_t *ptr) #elif defined(__APPLE__) ret = OSAtomicDecrement32(&ptr->value); #else - #error "Not supported on this platform." +# error "Not supported on this platform." #endif return ret; @@ -361,7 +379,8 @@ hg_atomic_xor32(hg_atomic_int32_t *ptr, hg_util_int32_t value) #elif defined(HG_UTIL_HAS_STDATOMIC_H) && !defined(HG_UTIL_HAS_OPA_PRIMITIVES_H) ret = atomic_fetch_xor_explicit(ptr, value, memory_order_acq_rel); #elif defined(__APPLE__) - ret = OSAtomicXor32Orig((uint32_t) value, (volatile uint32_t *) &ptr->value); + ret = + OSAtomicXor32Orig((uint32_t) value, (volatile uint32_t *) &ptr->value); #else do { ret = hg_atomic_get32(ptr); @@ -382,7 +401,8 @@ hg_atomic_and32(hg_atomic_int32_t *ptr, hg_util_int32_t value) #elif defined(HG_UTIL_HAS_STDATOMIC_H) && !defined(HG_UTIL_HAS_OPA_PRIMITIVES_H) ret = atomic_fetch_and_explicit(ptr, value, memory_order_acq_rel); #elif defined(__APPLE__) - ret = OSAtomicAnd32Orig((uint32_t) value, (volatile uint32_t *) &ptr->value); + ret = + OSAtomicAnd32Orig((uint32_t) value, (volatile uint32_t *) &ptr->value); #else do { ret = hg_atomic_get32(ptr); @@ -400,17 +420,18 @@ hg_atomic_cas32(hg_atomic_int32_t *ptr, hg_util_int32_t compare_value, hg_util_bool_t ret; #if defined(_WIN32) - ret = (compare_value == InterlockedCompareExchangeNoFence(&ptr->value, - swap_value, compare_value)); + ret = (compare_value == InterlockedCompareExchangeNoFence( + &ptr->value, swap_value, compare_value)); #elif defined(HG_UTIL_HAS_OPA_PRIMITIVES_H) - ret = (hg_util_bool_t) (compare_value == OPA_cas_int(ptr, compare_value, swap_value)); + ret = (hg_util_bool_t)( + compare_value == OPA_cas_int(ptr, compare_value, swap_value)); #elif defined(HG_UTIL_HAS_STDATOMIC_H) ret = atomic_compare_exchange_strong_explicit(ptr, &compare_value, swap_value, memory_order_acq_rel, memory_order_acquire); #elif defined(__APPLE__) ret = OSAtomicCompareAndSwap32(compare_value, swap_value, &ptr->value); #else - #error "Not supported on this platform." +# error "Not supported on this platform." #endif return ret; @@ -440,7 +461,7 @@ hg_atomic_set64(hg_atomic_int64_t *ptr, hg_util_int64_t value) #elif defined(__APPLE__) ptr->value = value; #else - #error "Not supported on this platform." +# error "Not supported on this platform." #endif } @@ -459,7 +480,7 @@ hg_atomic_get64(hg_atomic_int64_t *ptr) #elif defined(__APPLE__) ptr->value = value; #else - #error "Not supported on this platform." +# error "Not supported on this platform." #endif return ret; @@ -574,18 +595,19 @@ hg_atomic_cas64(hg_atomic_int64_t *ptr, hg_util_int64_t compare_value, hg_util_bool_t ret; #if defined(_WIN32) - ret = (compare_value == InterlockedCompareExchangeNoFence64(&ptr->value, - swap_value, compare_value)); + ret = (compare_value == InterlockedCompareExchangeNoFence64( + &ptr->value, swap_value, compare_value)); #elif defined(HG_UTIL_HAS_OPA_PRIMITIVES_H) - ret = (hg_util_bool_t) (compare_value == (hg_util_int64_t) OPA_cas_ptr(ptr, - (void *) compare_value, (void *) swap_value)); + ret = (hg_util_bool_t)( + compare_value == (hg_util_int64_t) OPA_cas_ptr( + ptr, (void *) compare_value, (void *) swap_value)); #elif defined(HG_UTIL_HAS_STDATOMIC_H) - ret = atomic_compare_exchange_strong_explicit(ptr, &compare_value, swap_value, - memory_order_acq_rel, memory_order_acquire); + ret = atomic_compare_exchange_strong_explicit(ptr, &compare_value, + swap_value, memory_order_acq_rel, memory_order_acquire); #elif defined(__APPLE__) ret = OSAtomicCompareAndSwap64(compare_value, swap_value, &ptr->value); #else - #error "Not supported on this platform." +# error "Not supported on this platform." #endif return ret; @@ -600,14 +622,11 @@ hg_atomic_fence() #elif defined(HG_UTIL_HAS_OPA_PRIMITIVES_H) OPA_read_write_barrier(); #elif defined(HG_UTIL_HAS_STDATOMIC_H) -#ifdef __INTEL_COMPILER -#else atomic_thread_fence(memory_order_acq_rel); -#endif #elif defined(__APPLE__) OSMemoryBarrier(); #else - #error "Not supported on this platform." +# error "Not supported on this platform." #endif } diff --git a/src/util/mercury_atomic_queue.c b/src/util/mercury_atomic_queue.c index 18b1af7e..4c6a8e2b 100644 --- a/src/util/mercury_atomic_queue.c +++ b/src/util/mercury_atomic_queue.c @@ -48,7 +48,7 @@ /****************/ /* From */ -#define powerof2(x) ((((x) - 1) & (x)) == 0) +#define powerof2(x) ((((x) -1) & (x)) == 0) /*---------------------------------------------------------------------------*/ struct hg_atomic_queue * @@ -56,17 +56,13 @@ hg_atomic_queue_alloc(unsigned int count) { struct hg_atomic_queue *hg_atomic_queue = NULL; - if (!powerof2(count)) { - HG_UTIL_LOG_ERROR("atomic queue size must be power of 2"); - goto done; - } + HG_UTIL_CHECK_ERROR_NORET( + !powerof2(count), done, "atomic queue size must be power of 2"); - hg_atomic_queue = malloc(sizeof(struct hg_atomic_queue) + - count * HG_ATOMIC_QUEUE_ELT_SIZE); - if (!hg_atomic_queue) { - HG_UTIL_LOG_ERROR("Could not allocate atomic queue"); - goto done; - } + hg_atomic_queue = malloc( + sizeof(struct hg_atomic_queue) + count * sizeof(hg_atomic_int64_t)); + HG_UTIL_CHECK_ERROR_NORET( + hg_atomic_queue == NULL, done, "Could not allocate atomic queue"); hg_atomic_queue->prod_size = hg_atomic_queue->cons_size = count; hg_atomic_queue->prod_mask = hg_atomic_queue->cons_mask = count - 1; diff --git a/src/util/mercury_atomic_queue.h b/src/util/mercury_atomic_queue.h index 8ea27413..73c0b153 100644 --- a/src/util/mercury_atomic_queue.h +++ b/src/util/mercury_atomic_queue.h @@ -42,38 +42,36 @@ #define MERCURY_ATOMIC_QUEUE_H #include "mercury_atomic.h" -#include "mercury_thread.h" +#include "mercury_mem.h" /*************************************/ /* Public Type and Struct Definition */ /*************************************/ -#define HG_UTIL_CACHE_ALIGNMENT 64 struct hg_atomic_queue { hg_atomic_int32_t prod_head; hg_atomic_int32_t prod_tail; - unsigned int prod_size; - unsigned int prod_mask; - hg_util_uint64_t drops; - hg_atomic_int32_t cons_head __attribute__((aligned(HG_UTIL_CACHE_ALIGNMENT))); + unsigned int prod_size; + unsigned int prod_mask; + hg_util_uint64_t drops; + hg_atomic_int32_t cons_head + __attribute__((aligned(HG_MEM_CACHE_LINE_SIZE))); hg_atomic_int32_t cons_tail; - unsigned int cons_size; - unsigned int cons_mask; - hg_atomic_int64_t *ring[1] __attribute__((aligned(HG_UTIL_CACHE_ALIGNMENT))); + unsigned int cons_size; + unsigned int cons_mask; + hg_atomic_int64_t ring[] __attribute__((aligned(HG_MEM_CACHE_LINE_SIZE))); }; /*****************/ /* Public Macros */ /*****************/ -#define HG_ATOMIC_QUEUE_ELT_SIZE sizeof(hg_atomic_int64_t) - #ifndef cpu_spinwait -# if defined(__x86_64__) || defined(__amd64__) -# define cpu_spinwait() asm volatile("pause\n": : :"memory"); -# else -# define cpu_spinwait(); -# endif +# if defined(__x86_64__) || defined(__amd64__) +# define cpu_spinwait() asm volatile("pause\n" : : : "memory"); +# else +# define cpu_spinwait() ; +# endif #endif /*********************/ @@ -91,7 +89,7 @@ extern "C" { * * \return pointer to allocated queue or NULL on failure */ -HG_UTIL_EXPORT struct hg_atomic_queue * +HG_UTIL_PUBLIC struct hg_atomic_queue * hg_atomic_queue_alloc(unsigned int count); /** @@ -99,7 +97,7 @@ hg_atomic_queue_alloc(unsigned int count); * * \param hg_atomic_queue [IN] pointer to queue */ -HG_UTIL_EXPORT void +HG_UTIL_PUBLIC void hg_atomic_queue_free(struct hg_atomic_queue *hg_atomic_queue); /** @@ -158,7 +156,6 @@ static HG_UTIL_INLINE int hg_atomic_queue_push(struct hg_atomic_queue *hg_atomic_queue, void *entry) { hg_util_int32_t prod_head, prod_next, cons_tail; - int ret = HG_UTIL_SUCCESS; do { prod_head = hg_atomic_get32(&hg_atomic_queue->prod_head); @@ -171,16 +168,14 @@ hg_atomic_queue_push(struct hg_atomic_queue *hg_atomic_queue, void *entry) cons_tail == hg_atomic_get32(&hg_atomic_queue->cons_tail)) { hg_atomic_queue->drops++; /* Full */ - ret = HG_UTIL_FAIL; - goto done; + return HG_UTIL_FAIL; } continue; } - } while (!hg_atomic_cas32(&hg_atomic_queue->prod_head, prod_head, - prod_next)); + } while ( + !hg_atomic_cas32(&hg_atomic_queue->prod_head, prod_head, prod_next)); - hg_atomic_set64((hg_atomic_int64_t *) &hg_atomic_queue->ring[prod_head], - (hg_util_int64_t) entry); + hg_atomic_set64(&hg_atomic_queue->ring[prod_head], (hg_util_int64_t) entry); /* * If there are other enqueues in progress @@ -192,8 +187,7 @@ hg_atomic_queue_push(struct hg_atomic_queue *hg_atomic_queue, void *entry) hg_atomic_set32(&hg_atomic_queue->prod_tail, prod_next); -done: - return ret; + return HG_UTIL_SUCCESS; } /*---------------------------------------------------------------------------*/ @@ -208,12 +202,11 @@ hg_atomic_queue_pop_mc(struct hg_atomic_queue *hg_atomic_queue) cons_next = (cons_head + 1) & (int) hg_atomic_queue->cons_mask; if (cons_head == hg_atomic_get32(&hg_atomic_queue->prod_tail)) - goto done; - } while (!hg_atomic_cas32(&hg_atomic_queue->cons_head, cons_head, - cons_next)); + return NULL; + } while ( + !hg_atomic_cas32(&hg_atomic_queue->cons_head, cons_head, cons_next)); - entry = (void *) hg_atomic_get64( - (hg_atomic_int64_t *) &hg_atomic_queue->ring[cons_head]); + entry = (void *) hg_atomic_get64(&hg_atomic_queue->ring[cons_head]); /* * If there are other dequeues in progress @@ -225,7 +218,6 @@ hg_atomic_queue_pop_mc(struct hg_atomic_queue *hg_atomic_queue) hg_atomic_set32(&hg_atomic_queue->cons_tail, cons_next); -done: return entry; } @@ -243,16 +235,14 @@ hg_atomic_queue_pop_sc(struct hg_atomic_queue *hg_atomic_queue) if (cons_head == prod_tail) /* Empty */ - goto done; + return NULL; hg_atomic_set32(&hg_atomic_queue->cons_head, cons_next); - entry = (void *) hg_atomic_get64( - (hg_atomic_int64_t *) &hg_atomic_queue->ring[cons_head]); + entry = (void *) hg_atomic_get64(&hg_atomic_queue->ring[cons_head]); hg_atomic_set32(&hg_atomic_queue->cons_tail, cons_next); -done: return entry; } @@ -261,17 +251,17 @@ static HG_UTIL_INLINE hg_util_bool_t hg_atomic_queue_is_empty(struct hg_atomic_queue *hg_atomic_queue) { return (hg_atomic_get32(&hg_atomic_queue->cons_head) == - hg_atomic_get32(&hg_atomic_queue->prod_tail)); + hg_atomic_get32(&hg_atomic_queue->prod_tail)); } /*---------------------------------------------------------------------------*/ static HG_UTIL_INLINE unsigned int hg_atomic_queue_count(struct hg_atomic_queue *hg_atomic_queue) { - return ((hg_atomic_queue->prod_size - + (unsigned int) hg_atomic_get32(&hg_atomic_queue->prod_tail) - - (unsigned int) hg_atomic_get32(&hg_atomic_queue->cons_tail)) - & hg_atomic_queue->prod_mask); + return ((hg_atomic_queue->prod_size + + (unsigned int) hg_atomic_get32(&hg_atomic_queue->prod_tail) - + (unsigned int) hg_atomic_get32(&hg_atomic_queue->cons_tail)) & + hg_atomic_queue->prod_mask); } #ifdef __cplusplus diff --git a/src/util/mercury_event.c b/src/util/mercury_event.c index 6100b6a5..42f45331 100644 --- a/src/util/mercury_event.c +++ b/src/util/mercury_event.c @@ -10,61 +10,62 @@ #include "mercury_event.h" +#include "mercury_util_error.h" + /*---------------------------------------------------------------------------*/ int hg_event_create(void) { - int fd = 0; + int fd = -1; #if defined(_WIN32) #elif defined(HG_UTIL_HAS_SYSEVENTFD_H) /* Create local signal event on self address */ fd = eventfd(0, EFD_NONBLOCK | EFD_SEMAPHORE); - if (fd == -1) { - HG_UTIL_LOG_ERROR("eventfd() failed (%s)", strerror(errno)); - goto done; - } + HG_UTIL_CHECK_ERROR_NORET( + fd == -1, done, "eventfd() failed (%s)", strerror(errno)); #elif defined(HG_UTIL_HAS_SYSEVENT_H) struct kevent kev; struct timespec timeout = {0, 0}; + int rc; /* Create kqueue */ fd = kqueue(); - if (fd == -1) { - HG_UTIL_LOG_ERROR("kqueue() failed (%s)", strerror(errno)); - goto done; - } + HG_UTIL_CHECK_ERROR_NORET( + fd == -1, done, "kqueue() failed (%s)", strerror(errno)); EV_SET(&kev, HG_EVENT_IDENT, EVFILT_USER, EV_ADD | EV_CLEAR, 0, 0, NULL); /* Add user-defined event to kqueue */ - if (kevent(fd, &kev, 1, NULL, 0, &timeout) == -1) { - HG_UTIL_LOG_ERROR("kevent() failed (%s)", strerror(errno)); - hg_event_destroy(fd); - fd = 0; - goto done; - } + rc = kevent(fd, &kev, 1, NULL, 0, &timeout); + HG_UTIL_CHECK_ERROR_NORET( + rc == -1, error, "kevent() failed (%s)", strerror(errno)); #else #endif done: return fd; + +#if defined(HG_UTIL_HAS_SYSEVENT_H) +error: + hg_event_destroy(fd); + + return -1; +#endif } /*---------------------------------------------------------------------------*/ int hg_event_destroy(int fd) { - int ret = HG_UTIL_SUCCESS; + int ret = HG_UTIL_SUCCESS, rc; #if defined(_WIN32) #else - if (close(fd) == -1) { - HG_UTIL_LOG_ERROR("close() failed (%s)", strerror(errno)); - ret = HG_UTIL_FAIL; - goto done; - } + rc = close(fd); + HG_UTIL_CHECK_ERROR(rc == -1, done, ret, HG_UTIL_FAIL, + "close() failed (%s)", strerror(errno)); #endif done: return ret; diff --git a/src/util/mercury_event.h b/src/util/mercury_event.h index 6b2d77d8..48175b11 100644 --- a/src/util/mercury_event.h +++ b/src/util/mercury_event.h @@ -12,24 +12,22 @@ #define MERCURY_EVENT_H #include "mercury_util_config.h" -#include "mercury_util_error.h" #ifdef _WIN32 #else -# include -# include -# include -# if defined(HG_UTIL_HAS_SYSEVENTFD_H) -# include -# ifndef HG_UTIL_HAS_EVENTFD_T +# include +# include +# include +# if defined(HG_UTIL_HAS_SYSEVENTFD_H) +# include +# ifndef HG_UTIL_HAS_EVENTFD_T typedef uint64_t eventfd_t; -# endif -# elif defined(HG_UTIL_HAS_SYSEVENT_H) -# include -/* User-defined ident */ -# define HG_EVENT_IDENT 42 -# endif +# endif +# elif defined(HG_UTIL_HAS_SYSEVENT_H) +# include +# define HG_EVENT_IDENT 42 /* User-defined ident */ +# endif #endif /** @@ -46,7 +44,7 @@ extern "C" { * * \return file descriptor on success or negative on failure */ -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_event_create(void); /** @@ -56,7 +54,7 @@ hg_event_create(void); * * \return Non-negative on success or negative on failure */ -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_event_destroy(int fd); /** @@ -84,111 +82,99 @@ hg_event_get(int fd, hg_util_bool_t *notified); #if defined(_WIN32) /* TODO */ #elif defined(HG_UTIL_HAS_SYSEVENTFD_H) +# ifdef HG_UTIL_HAS_EVENTFD_T +static HG_UTIL_INLINE int +hg_event_set(int fd) +{ + return (eventfd_write(fd, 1) == 0) ? HG_UTIL_SUCCESS : HG_UTIL_FAIL; +} +# else static HG_UTIL_INLINE int hg_event_set(int fd) { - int ret = HG_UTIL_SUCCESS; eventfd_t count = 1; + ssize_t s = write(fd, &count, sizeof(eventfd_t)); -#ifdef HG_UTIL_HAS_EVENTFD_T - if (eventfd_write(fd, count) == -1) { -#else - ssize_t s; - - s = write(fd, &count, sizeof(eventfd_t)); - if (s != sizeof(eventfd_t)) { -#endif - if (errno == EAGAIN) - goto done; - HG_UTIL_LOG_ERROR("write() failed (%s)", strerror(errno)); - ret = HG_UTIL_FAIL; - goto done; - } - -done: - return ret; + return (s == sizeof(eventfd_t)) ? HG_UTIL_SUCCESS : HG_UTIL_FAIL; } +# endif #elif defined(HG_UTIL_HAS_SYSEVENT_H) static HG_UTIL_INLINE int hg_event_set(int fd) { - int ret = HG_UTIL_SUCCESS; struct kevent kev; struct timespec timeout = {0, 0}; + int rc; EV_SET(&kev, HG_EVENT_IDENT, EVFILT_USER, 0, NOTE_TRIGGER, 0, NULL); /* Trigger user-defined event */ - if (kevent(fd, &kev, 1, NULL, 0, &timeout) == -1) { - HG_UTIL_LOG_ERROR("kevent() failed (%s)", strerror(errno)); - ret = HG_UTIL_FAIL; - goto done; - } + rc = kevent(fd, &kev, 1, NULL, 0, &timeout); -done: - return ret; + return (rc == -1) ? HG_UTIL_FAIL : HG_UTIL_SUCCESS; } #else -# error "Not supported on this platform." +# error "Not supported on this platform." #endif /*---------------------------------------------------------------------------*/ #if defined(_WIN32) #elif defined(HG_UTIL_HAS_SYSEVENTFD_H) +# ifdef HG_UTIL_HAS_EVENTFD_T static HG_UTIL_INLINE int hg_event_get(int fd, hg_util_bool_t *signaled) { - int ret = HG_UTIL_SUCCESS; - hg_util_bool_t event_signal = HG_UTIL_FALSE; eventfd_t count = 0; -#ifdef HG_UTIL_HAS_EVENTFD_T - if (eventfd_read(fd, &count) == -1) { -#else - ssize_t s; + if ((eventfd_read(fd, &count) == 0) && count) + *signaled = HG_UTIL_TRUE; + else { + if (errno == EAGAIN) + *signaled = HG_UTIL_FALSE; + else + return HG_UTIL_FAIL; + } - s = read(fd, &count, sizeof(eventfd_t)); - if (s != sizeof(eventfd_t)) { -#endif + return HG_UTIL_SUCCESS; +} +# else +static HG_UTIL_INLINE int +hg_event_get(int fd, hg_util_bool_t *signaled) +{ + eventfd_t count = 0; + ssize_t s = read(fd, &count, sizeof(eventfd_t)); + if ((s == sizeof(eventfd_t)) && count) + *signaled = HG_UTIL_TRUE; + else { if (errno == EAGAIN) - goto done; - HG_UTIL_LOG_ERROR("read() failed (%s)", strerror(errno)); - ret = HG_UTIL_FAIL; - goto done; + *signaled = HG_UTIL_FALSE; + else + return HG_UTIL_FAIL; } - event_signal = HG_UTIL_TRUE; -done: - if (signaled && ret != HG_UTIL_FAIL) - *signaled = event_signal; - return ret; + return HG_UTIL_SUCCESS; } +# endif #elif defined(HG_UTIL_HAS_SYSEVENT_H) static HG_UTIL_INLINE int hg_event_get(int fd, hg_util_bool_t *signaled) { - int ret = HG_UTIL_SUCCESS; - hg_util_bool_t event_signal = HG_UTIL_FALSE; struct kevent kev; int nfds; struct timespec timeout = {0, 0}; /* Check user-defined event */ nfds = kevent(fd, NULL, 0, &kev, 1, &timeout); - if (nfds == -1) { - HG_UTIL_LOG_ERROR("kevent() failed (%s)", strerror(errno)); - ret = HG_UTIL_FAIL; - goto done; - } - if (nfds > 0 && kev.ident == HG_EVENT_IDENT) - event_signal = HG_UTIL_TRUE; + if (nfds == -1) + return HG_UTIL_FAIL; - if (signaled) *signaled = event_signal; + *signaled = ((nfds > 0) && (kev.ident == HG_EVENT_IDENT)) ? HG_UTIL_TRUE + : HG_UTIL_FALSE; -done: - return ret; + return HG_UTIL_SUCCESS; } #else +# error "Not supported on this platform." #endif #ifdef __cplusplus diff --git a/src/util/mercury_hash_table.c b/src/util/mercury_hash_table.c index 42211f0b..d7d14dfc 100644 --- a/src/util/mercury_hash_table.c +++ b/src/util/mercury_hash_table.c @@ -2,19 +2,19 @@ Copyright (c) 2005-2008, Simon Howard -Permission to use, copy, modify, and/or distribute this software -for any purpose with or without fee is hereby granted, provided -that the above copyright notice and this permission notice appear -in all copies. - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL -WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE -AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR -CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, -NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +Permission to use, copy, modify, and/or distribute this software +for any purpose with or without fee is hereby granted, provided +that the above copyright notice and this permission notice appear +in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL +WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE +AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR +CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ @@ -26,20 +26,20 @@ CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #include struct hg_hash_table_entry { - hg_hash_table_key_t key; - hg_hash_table_value_t value; - hg_hash_table_entry_t *next; + hg_hash_table_key_t key; + hg_hash_table_value_t value; + hg_hash_table_entry_t *next; }; struct hg_hash_table { - hg_hash_table_entry_t **table; - unsigned int table_size; - hg_hash_table_hash_func_t hash_func; - hg_hash_table_equal_func_t equal_func; - hg_hash_table_key_free_func_t key_free_func; - hg_hash_table_value_free_func_t value_free_func; - unsigned int entries; - unsigned int prime_index; + hg_hash_table_entry_t **table; + unsigned int table_size; + hg_hash_table_hash_func_t hash_func; + hg_hash_table_equal_func_t equal_func; + hg_hash_table_key_free_func_t key_free_func; + hg_hash_table_value_free_func_t value_free_func; + unsigned int entries; + unsigned int prime_index; }; /* This is a set of good hash table prime numbers, from: @@ -48,445 +48,479 @@ struct hg_hash_table { * possible from the nearest powers of two. */ static const unsigned int hash_table_primes[] = { - 193, 389, 769, 1543, 3079, 6151, 12289, 24593, 49157, 98317, - 196613, 393241, 786433, 1572869, 3145739, 6291469, - 12582917, 25165843, 50331653, 100663319, 201326611, - 402653189, 805306457, 1610612741, + 193, + 389, + 769, + 1543, + 3079, + 6151, + 12289, + 24593, + 49157, + 98317, + 196613, + 393241, + 786433, + 1572869, + 3145739, + 6291469, + 12582917, + 25165843, + 50331653, + 100663319, + 201326611, + 402653189, + 805306457, + 1610612741, }; -static const unsigned int hash_table_num_primes - = sizeof(hash_table_primes) / sizeof(int); +static const unsigned int hash_table_num_primes = + sizeof(hash_table_primes) / sizeof(int); /* Internal function used to allocate the table on hash table creation * and when enlarging the table */ -static int hash_table_allocate_table(hg_hash_table_t *hash_table) +static int +hash_table_allocate_table(hg_hash_table_t *hash_table) { - unsigned int new_table_size; + unsigned int new_table_size; - /* Determine the table size based on the current prime index. - * An attempt is made here to ensure sensible behavior if the - * maximum prime is exceeded, but in practice other things are - * likely to break long before that happens. */ + /* Determine the table size based on the current prime index. + * An attempt is made here to ensure sensible behavior if the + * maximum prime is exceeded, but in practice other things are + * likely to break long before that happens. */ - if (hash_table->prime_index < hash_table_num_primes) { - new_table_size = hash_table_primes[hash_table->prime_index]; - } else { - new_table_size = hash_table->entries * 10; - } + if (hash_table->prime_index < hash_table_num_primes) { + new_table_size = hash_table_primes[hash_table->prime_index]; + } else { + new_table_size = hash_table->entries * 10; + } - hash_table->table_size = new_table_size; + hash_table->table_size = new_table_size; - /* Allocate the table and initialise to NULL for all entries */ + /* Allocate the table and initialise to NULL for all entries */ - hash_table->table = (hg_hash_table_entry_t **) calloc(hash_table->table_size, - sizeof(hg_hash_table_entry_t *)); + hash_table->table = (hg_hash_table_entry_t **) calloc( + hash_table->table_size, sizeof(hg_hash_table_entry_t *)); - return hash_table->table != NULL; + return hash_table->table != NULL; } /* Free an entry, calling the free functions if there are any registered */ -static void hash_table_free_entry(hg_hash_table_t *hash_table, hg_hash_table_entry_t *entry) +static void +hash_table_free_entry(hg_hash_table_t *hash_table, hg_hash_table_entry_t *entry) { - /* If there is a function registered for freeing keys, use it to free - * the key */ - - if (hash_table->key_free_func != NULL) { - hash_table->key_free_func(entry->key); - } - - /* Likewise with the value */ - - if (hash_table->value_free_func != NULL) { - hash_table->value_free_func(entry->value); - } - - /* Free the data structure */ - - free(entry); + /* If there is a function registered for freeing keys, use it to free + * the key */ + + if (hash_table->key_free_func != NULL) { + hash_table->key_free_func(entry->key); + } + + /* Likewise with the value */ + + if (hash_table->value_free_func != NULL) { + hash_table->value_free_func(entry->value); + } + + /* Free the data structure */ + + free(entry); } -hg_hash_table_t *hg_hash_table_new(hg_hash_table_hash_func_t hash_func, - hg_hash_table_equal_func_t equal_func) +hg_hash_table_t * +hg_hash_table_new( + hg_hash_table_hash_func_t hash_func, hg_hash_table_equal_func_t equal_func) { - hg_hash_table_t *hash_table; + hg_hash_table_t *hash_table; + + /* Allocate a new hash table structure */ - /* Allocate a new hash table structure */ - - hash_table = (hg_hash_table_t *) malloc(sizeof(hg_hash_table_t)); + hash_table = (hg_hash_table_t *) malloc(sizeof(hg_hash_table_t)); - if (hash_table == NULL) { - return NULL; - } - - hash_table->hash_func = hash_func; - hash_table->equal_func = equal_func; - hash_table->key_free_func = NULL; - hash_table->value_free_func = NULL; - hash_table->entries = 0; - hash_table->prime_index = 0; + if (hash_table == NULL) { + return NULL; + } - /* Allocate the table */ + hash_table->hash_func = hash_func; + hash_table->equal_func = equal_func; + hash_table->key_free_func = NULL; + hash_table->value_free_func = NULL; + hash_table->entries = 0; + hash_table->prime_index = 0; - if (!hash_table_allocate_table(hash_table)) { - free(hash_table); + /* Allocate the table */ - return NULL; - } + if (!hash_table_allocate_table(hash_table)) { + free(hash_table); - return hash_table; + return NULL; + } + + return hash_table; } -void hg_hash_table_free(hg_hash_table_t *hash_table) +void +hg_hash_table_free(hg_hash_table_t *hash_table) { - hg_hash_table_entry_t *rover; - hg_hash_table_entry_t *next; - unsigned int i; - - /* Free all entries in all chains */ - - for (i=0; itable_size; ++i) { - rover = hash_table->table[i]; - while (rover != NULL) { - next = rover->next; - hash_table_free_entry(hash_table, rover); - rover = next; - } - } - - /* Free the table */ - - free(hash_table->table); - - /* Free the hash table structure */ - - free(hash_table); + hg_hash_table_entry_t *rover; + hg_hash_table_entry_t *next; + unsigned int i; + + /* Free all entries in all chains */ + + for (i = 0; i < hash_table->table_size; ++i) { + rover = hash_table->table[i]; + while (rover != NULL) { + next = rover->next; + hash_table_free_entry(hash_table, rover); + rover = next; + } + } + + /* Free the table */ + + free(hash_table->table); + + /* Free the hash table structure */ + + free(hash_table); } -void hg_hash_table_register_free_functions(hg_hash_table_t *hash_table, - hg_hash_table_key_free_func_t key_free_func, - hg_hash_table_value_free_func_t value_free_func) +void +hg_hash_table_register_free_functions(hg_hash_table_t *hash_table, + hg_hash_table_key_free_func_t key_free_func, + hg_hash_table_value_free_func_t value_free_func) { - hash_table->key_free_func = key_free_func; - hash_table->value_free_func = value_free_func; + hash_table->key_free_func = key_free_func; + hash_table->value_free_func = value_free_func; } - -static int hash_table_enlarge(hg_hash_table_t *hash_table) +static int +hash_table_enlarge(hg_hash_table_t *hash_table) { - hg_hash_table_entry_t **old_table; - unsigned int old_table_size; - unsigned int old_prime_index; - hg_hash_table_entry_t *rover; - hg_hash_table_entry_t *next; - unsigned int entry_index; - unsigned int i; - - /* Store a copy of the old table */ - - old_table = hash_table->table; - old_table_size = hash_table->table_size; - old_prime_index = hash_table->prime_index; - - /* Allocate a new, larger table */ - - ++hash_table->prime_index; - - if (!hash_table_allocate_table(hash_table)) { - - /* Failed to allocate the new table */ - - hash_table->table = old_table; - hash_table->table_size = old_table_size; - hash_table->prime_index = old_prime_index; - - return 0; - } - - /* Link all entries from all chains into the new table */ - - for (i=0; inext; - - /* Find the index into the new table */ - - entry_index = hash_table->hash_func(rover->key) % hash_table->table_size; - - /* Link this entry into the chain */ - - rover->next = hash_table->table[entry_index]; - hash_table->table[entry_index] = rover; - - /* Advance to next in the chain */ - - rover = next; - } - } - - /* Free the old table */ - - free(old_table); - - return 1; + hg_hash_table_entry_t **old_table; + unsigned int old_table_size; + unsigned int old_prime_index; + hg_hash_table_entry_t *rover; + hg_hash_table_entry_t *next; + unsigned int entry_index; + unsigned int i; + + /* Store a copy of the old table */ + + old_table = hash_table->table; + old_table_size = hash_table->table_size; + old_prime_index = hash_table->prime_index; + + /* Allocate a new, larger table */ + + ++hash_table->prime_index; + + if (!hash_table_allocate_table(hash_table)) { + + /* Failed to allocate the new table */ + + hash_table->table = old_table; + hash_table->table_size = old_table_size; + hash_table->prime_index = old_prime_index; + + return 0; + } + + /* Link all entries from all chains into the new table */ + + for (i = 0; i < old_table_size; ++i) { + rover = old_table[i]; + + while (rover != NULL) { + next = rover->next; + + /* Find the index into the new table */ + + entry_index = + hash_table->hash_func(rover->key) % hash_table->table_size; + + /* Link this entry into the chain */ + + rover->next = hash_table->table[entry_index]; + hash_table->table[entry_index] = rover; + + /* Advance to next in the chain */ + + rover = next; + } + } + + /* Free the old table */ + + free(old_table); + + return 1; } -int hg_hash_table_insert(hg_hash_table_t *hash_table, hg_hash_table_key_t key, hg_hash_table_value_t value) +int +hg_hash_table_insert(hg_hash_table_t *hash_table, hg_hash_table_key_t key, + hg_hash_table_value_t value) { - hg_hash_table_entry_t *rover; - hg_hash_table_entry_t *newentry; - unsigned int entry_index; - - /* If there are too many items in the table with respect to the table - * size, the number of hash collisions increases and performance - * decreases. Enlarge the table size to prevent this happening */ + hg_hash_table_entry_t *rover; + hg_hash_table_entry_t *newentry; + unsigned int entry_index; + + /* If there are too many items in the table with respect to the table + * size, the number of hash collisions increases and performance + * decreases. Enlarge the table size to prevent this happening */ + + if ((hash_table->entries * 3) / hash_table->table_size > 0) { + + /* Table is more than 1/3 full */ - if ((hash_table->entries * 3) / hash_table->table_size > 0) { - - /* Table is more than 1/3 full */ + if (!hash_table_enlarge(hash_table)) { - if (!hash_table_enlarge(hash_table)) { + /* Failed to enlarge the table */ - /* Failed to enlarge the table */ + return 0; + } + } - return 0; - } - } + /* Generate the hash of the key and hence the index into the table */ - /* Generate the hash of the key and hence the index into the table */ + entry_index = hash_table->hash_func(key) % hash_table->table_size; - entry_index = hash_table->hash_func(key) % hash_table->table_size; + /* Traverse the chain at this location and look for an existing + * entry with the same key */ - /* Traverse the chain at this location and look for an existing - * entry with the same key */ + rover = hash_table->table[entry_index]; - rover = hash_table->table[entry_index]; + while (rover != NULL) { + if (hash_table->equal_func(rover->key, key) != 0) { - while (rover != NULL) { - if (hash_table->equal_func(rover->key, key) != 0) { + /* Same key: overwrite this entry with new data */ - /* Same key: overwrite this entry with new data */ + /* If there is a value free function, free the old data + * before adding in the new data */ - /* If there is a value free function, free the old data - * before adding in the new data */ + if (hash_table->value_free_func != NULL) { + hash_table->value_free_func(rover->value); + } - if (hash_table->value_free_func != NULL) { - hash_table->value_free_func(rover->value); - } + /* Same with the key: use the new key value and free + * the old one */ - /* Same with the key: use the new key value and free - * the old one */ + if (hash_table->key_free_func != NULL) { + hash_table->key_free_func(rover->key); + } - if (hash_table->key_free_func != NULL) { - hash_table->key_free_func(rover->key); - } + rover->key = key; + rover->value = value; - rover->key = key; - rover->value = value; + /* Finished */ - /* Finished */ - - return 1; - } - rover = rover->next; - } - - /* Not in the hash table yet. Create a new entry */ + return 1; + } + rover = rover->next; + } - newentry = (hg_hash_table_entry_t *) malloc(sizeof(hg_hash_table_entry_t)); + /* Not in the hash table yet. Create a new entry */ - if (newentry == NULL) { - return 0; - } + newentry = (hg_hash_table_entry_t *) malloc(sizeof(hg_hash_table_entry_t)); - newentry->key = key; - newentry->value = value; + if (newentry == NULL) { + return 0; + } - /* Link into the list */ + newentry->key = key; + newentry->value = value; - newentry->next = hash_table->table[entry_index]; - hash_table->table[entry_index] = newentry; + /* Link into the list */ - /* Maintain the count of the number of entries */ + newentry->next = hash_table->table[entry_index]; + hash_table->table[entry_index] = newentry; - ++hash_table->entries; + /* Maintain the count of the number of entries */ - /* Added successfully */ + ++hash_table->entries; - return 1; + /* Added successfully */ + + return 1; } -hg_hash_table_value_t hg_hash_table_lookup(hg_hash_table_t *hash_table, hg_hash_table_key_t key) +hg_hash_table_value_t +hg_hash_table_lookup(hg_hash_table_t *hash_table, hg_hash_table_key_t key) { - hg_hash_table_entry_t *rover; - unsigned int entry_index; + hg_hash_table_entry_t *rover; + unsigned int entry_index; + + /* Generate the hash of the key and hence the index into the table */ - /* Generate the hash of the key and hence the index into the table */ - - entry_index = hash_table->hash_func(key) % hash_table->table_size; + entry_index = hash_table->hash_func(key) % hash_table->table_size; - /* Walk the chain at this index until the corresponding entry is - * found */ + /* Walk the chain at this index until the corresponding entry is + * found */ - rover = hash_table->table[entry_index]; + rover = hash_table->table[entry_index]; - while (rover != NULL) { - if (hash_table->equal_func(key, rover->key) != 0) { + while (rover != NULL) { + if (hash_table->equal_func(key, rover->key) != 0) { - /* Found the entry. Return the data. */ + /* Found the entry. Return the data. */ - return rover->value; - } - rover = rover->next; - } + return rover->value; + } + rover = rover->next; + } - /* Not found */ + /* Not found */ - return HG_HASH_TABLE_NULL; + return HG_HASH_TABLE_NULL; } -int hg_hash_table_remove(hg_hash_table_t *hash_table, hg_hash_table_key_t key) +int +hg_hash_table_remove(hg_hash_table_t *hash_table, hg_hash_table_key_t key) { - hg_hash_table_entry_t **rover; - hg_hash_table_entry_t *entry; - unsigned int entry_index; - int result; + hg_hash_table_entry_t **rover; + hg_hash_table_entry_t *entry; + unsigned int entry_index; + int result; + + /* Generate the hash of the key and hence the index into the table */ - /* Generate the hash of the key and hence the index into the table */ - - entry_index = hash_table->hash_func(key) % hash_table->table_size; + entry_index = hash_table->hash_func(key) % hash_table->table_size; - /* Rover points at the pointer which points at the current entry - * in the chain being inspected. ie. the entry in the table, or - * the "next" pointer of the previous entry in the chain. This - * allows us to unlink the entry when we find it. */ + /* Rover points at the pointer which points at the current entry + * in the chain being inspected. ie. the entry in the table, or + * the "next" pointer of the previous entry in the chain. This + * allows us to unlink the entry when we find it. */ - result = 0; - rover = &hash_table->table[entry_index]; + result = 0; + rover = &hash_table->table[entry_index]; - while (*rover != NULL) { + while (*rover != NULL) { - if (hash_table->equal_func(key, (*rover)->key) != 0) { + if (hash_table->equal_func(key, (*rover)->key) != 0) { - /* This is the entry to remove */ + /* This is the entry to remove */ - entry = *rover; + entry = *rover; - /* Unlink from the list */ + /* Unlink from the list */ - *rover = entry->next; + *rover = entry->next; - /* Destroy the entry structure */ + /* Destroy the entry structure */ - hash_table_free_entry(hash_table, entry); + hash_table_free_entry(hash_table, entry); - /* Track count of entries */ + /* Track count of entries */ - --hash_table->entries; + --hash_table->entries; - result = 1; + result = 1; - break; - } - - /* Advance to the next entry */ + break; + } - rover = &((*rover)->next); - } + /* Advance to the next entry */ - return result; + rover = &((*rover)->next); + } + + return result; } -unsigned int hg_hash_table_num_entries(hg_hash_table_t *hash_table) +unsigned int +hg_hash_table_num_entries(hg_hash_table_t *hash_table) { - return hash_table->entries; + return hash_table->entries; } -void hg_hash_table_iterate(hg_hash_table_t *hash_table, hg_hash_table_iter_t *iterator) +void +hg_hash_table_iterate( + hg_hash_table_t *hash_table, hg_hash_table_iter_t *iterator) { - unsigned int chain; + unsigned int chain; - iterator->hash_table = hash_table; + iterator->hash_table = hash_table; - /* Default value of next if no entries are found. */ + /* Default value of next if no entries are found. */ - iterator->next_entry = NULL; + iterator->next_entry = NULL; - /* Find the first entry */ + /* Find the first entry */ - for (chain=0; chaintable_size; ++chain) { + for (chain = 0; chain < hash_table->table_size; ++chain) { - if (hash_table->table[chain] != NULL) { - iterator->next_entry = hash_table->table[chain]; - iterator->next_chain = chain; - break; - } - } + if (hash_table->table[chain] != NULL) { + iterator->next_entry = hash_table->table[chain]; + iterator->next_chain = chain; + break; + } + } } -int hg_hash_table_iter_has_more(hg_hash_table_iter_t *iterator) +int +hg_hash_table_iter_has_more(hg_hash_table_iter_t *iterator) { - return iterator->next_entry != NULL; + return iterator->next_entry != NULL; } -hg_hash_table_value_t hg_hash_table_iter_next(hg_hash_table_iter_t *iterator) +hg_hash_table_value_t +hg_hash_table_iter_next(hg_hash_table_iter_t *iterator) { - hg_hash_table_entry_t *current_entry; - hg_hash_table_t *hash_table; - hg_hash_table_value_t result; - unsigned int chain; + hg_hash_table_entry_t *current_entry; + hg_hash_table_t *hash_table; + hg_hash_table_value_t result; + unsigned int chain; - hash_table = iterator->hash_table; + hash_table = iterator->hash_table; - /* No more entries? */ + /* No more entries? */ - if (iterator->next_entry == NULL) { - return HG_HASH_TABLE_NULL; - } + if (iterator->next_entry == NULL) { + return HG_HASH_TABLE_NULL; + } - /* Result is immediately available */ + /* Result is immediately available */ - current_entry = iterator->next_entry; - result = current_entry->value; + current_entry = iterator->next_entry; + result = current_entry->value; - /* Find the next entry */ + /* Find the next entry */ - if (current_entry->next != NULL) { + if (current_entry->next != NULL) { - /* Next entry in current chain */ + /* Next entry in current chain */ - iterator->next_entry = current_entry->next; + iterator->next_entry = current_entry->next; - } else { + } else { - /* None left in this chain, so advance to the next chain */ + /* None left in this chain, so advance to the next chain */ - chain = iterator->next_chain + 1; + chain = iterator->next_chain + 1; - /* Default value if no next chain found */ + /* Default value if no next chain found */ - iterator->next_entry = NULL; + iterator->next_entry = NULL; - while (chain < hash_table->table_size) { + while (chain < hash_table->table_size) { - /* Is there anything in this chain? */ + /* Is there anything in this chain? */ - if (hash_table->table[chain] != NULL) { - iterator->next_entry = hash_table->table[chain]; - break; - } + if (hash_table->table[chain] != NULL) { + iterator->next_entry = hash_table->table[chain]; + break; + } - /* Try the next chain */ + /* Try the next chain */ - ++chain; - } + ++chain; + } - iterator->next_chain = chain; - } + iterator->next_chain = chain; + } - return result; + return result; } - diff --git a/src/util/mercury_hash_table.h b/src/util/mercury_hash_table.h index 3b4882bc..619857b0 100644 --- a/src/util/mercury_hash_table.h +++ b/src/util/mercury_hash_table.h @@ -2,19 +2,19 @@ Copyright (c) 2005-2008, Simon Howard -Permission to use, copy, modify, and/or distribute this software -for any purpose with or without fee is hereby granted, provided -that the above copyright notice and this permission notice appear -in all copies. - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL -WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE -AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR -CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, -NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +Permission to use, copy, modify, and/or distribute this software +for any purpose with or without fee is hereby granted, provided +that the above copyright notice and this permission notice appear +in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL +WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE +AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR +CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ @@ -42,7 +42,6 @@ CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. * \ref hg_hash_table_iter_next and \ref hg_hash_table_iter_has_more. */ - #ifndef HG_HASH_TABLE_H #define HG_HASH_TABLE_H @@ -52,7 +51,7 @@ CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. extern "C" { #endif -/** +/** * A hash table structure. */ @@ -89,7 +88,7 @@ typedef void *hg_hash_table_value_t; struct hg_hash_table_iter { hg_hash_table_t *hash_table; hg_hash_table_entry_t *next_entry; - unsigned int next_chain; + unsigned int next_chain; }; /** @@ -115,17 +114,18 @@ typedef unsigned int (*hg_hash_table_hash_func_t)(hg_hash_table_key_t value); * not equal. */ -typedef int (*hg_hash_table_equal_func_t)(hg_hash_table_key_t value1, hg_hash_table_key_t value2); +typedef int (*hg_hash_table_equal_func_t)( + hg_hash_table_key_t value1, hg_hash_table_key_t value2); /** - * Type of function used to free keys when entries are removed from a + * Type of function used to free keys when entries are removed from a * hash table. */ typedef void (*hg_hash_table_key_free_func_t)(hg_hash_table_key_t value); /** - * Type of function used to free values when entries are removed from a + * Type of function used to free values when entries are removed from a * hash table. */ @@ -142,16 +142,16 @@ typedef void (*hg_hash_table_value_free_func_t)(hg_hash_table_value_t value); * was not possible to allocate the new hash * table. */ -HG_UTIL_EXPORT hg_hash_table_t * -hg_hash_table_new(hg_hash_table_hash_func_t hash_func, - hg_hash_table_equal_func_t equal_func); +HG_UTIL_PUBLIC hg_hash_table_t * +hg_hash_table_new( + hg_hash_table_hash_func_t hash_func, hg_hash_table_equal_func_t equal_func); /** * Destroy a hash table. * * \param hash_table The hash table to destroy. */ -HG_UTIL_EXPORT void +HG_UTIL_PUBLIC void hg_hash_table_free(hg_hash_table_t *hash_table); /** @@ -162,13 +162,13 @@ hg_hash_table_free(hg_hash_table_t *hash_table); * \param key_free_func Function used to free keys. * \param value_free_func Function used to free values. */ -HG_UTIL_EXPORT void +HG_UTIL_PUBLIC void hg_hash_table_register_free_functions(hg_hash_table_t *hash_table, - hg_hash_table_key_free_func_t key_free_func, - hg_hash_table_value_free_func_t value_free_func); + hg_hash_table_key_free_func_t key_free_func, + hg_hash_table_value_free_func_t value_free_func); /** - * Insert a value into a hash table, overwriting any existing entry + * Insert a value into a hash table, overwriting any existing entry * using the same key. * * \param hash_table The hash table. @@ -178,10 +178,9 @@ hg_hash_table_register_free_functions(hg_hash_table_t *hash_table, * or zero if it was not possible to allocate * memory for the new entry. */ -HG_UTIL_EXPORT int -hg_hash_table_insert(hg_hash_table_t *hash_table, - hg_hash_table_key_t key, - hg_hash_table_value_t value); +HG_UTIL_PUBLIC int +hg_hash_table_insert(hg_hash_table_t *hash_table, hg_hash_table_key_t key, + hg_hash_table_value_t value); /** * Look up a value in a hash table by key. @@ -191,9 +190,8 @@ hg_hash_table_insert(hg_hash_table_t *hash_table, * \return The value, or \ref HASH_TABLE_NULL if there * is no value with that key in the hash table. */ -HG_UTIL_EXPORT hg_hash_table_value_t -hg_hash_table_lookup(hg_hash_table_t *hash_table, - hg_hash_table_key_t key); +HG_UTIL_PUBLIC hg_hash_table_value_t +hg_hash_table_lookup(hg_hash_table_t *hash_table, hg_hash_table_key_t key); /** * Remove a value from a hash table. @@ -203,16 +201,16 @@ hg_hash_table_lookup(hg_hash_table_t *hash_table, * \return Non-zero if a key was removed, or zero if the * specified key was not found in the hash table. */ -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_hash_table_remove(hg_hash_table_t *hash_table, hg_hash_table_key_t key); -/** +/** * Retrieve the number of entries in a hash table. * * \param hash_table The hash table. * \return The number of entries in the hash table. */ -HG_UTIL_EXPORT unsigned int +HG_UTIL_PUBLIC unsigned int hg_hash_table_num_entries(hg_hash_table_t *hash_table); /** @@ -222,19 +220,18 @@ hg_hash_table_num_entries(hg_hash_table_t *hash_table); * \param iter Pointer to an iterator structure to * initialise. */ -HG_UTIL_EXPORT void +HG_UTIL_PUBLIC void hg_hash_table_iterate(hg_hash_table_t *hash_table, hg_hash_table_iter_t *iter); /** - * Determine if there are more keys in the hash table to iterate - * over. + * Determine if there are more keys in the hash table to iterate over. * * \param iterator The hash table iterator. * \return Zero if there are no more values to iterate - * over, non-zero if there are more values to + * over, non-zero if there are more values to * iterate over. */ -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_hash_table_iter_has_more(hg_hash_table_iter_t *iterator); /** @@ -245,7 +242,7 @@ hg_hash_table_iter_has_more(hg_hash_table_iter_t *iterator); * \ref HG_HASH_TABLE_NULL if there are no more * keys to iterate over. */ -HG_UTIL_EXPORT hg_hash_table_value_t +HG_UTIL_PUBLIC hg_hash_table_value_t hg_hash_table_iter_next(hg_hash_table_iter_t *iterator); #ifdef __cplusplus @@ -253,4 +250,3 @@ hg_hash_table_iter_next(hg_hash_table_iter_t *iterator); #endif #endif /* HG_HASH_TABLE_H */ - diff --git a/src/util/mercury_list.h b/src/util/mercury_list.h index 76fb5d16..5a29e4af 100644 --- a/src/util/mercury_list.h +++ b/src/util/mercury_list.h @@ -43,78 +43,84 @@ #ifndef MERCURY_LIST_H #define MERCURY_LIST_H -#define HG_LIST_HEAD_INITIALIZER(name) { NULL } +#define HG_LIST_HEAD_INITIALIZER(name) \ + { \ + NULL \ + } -#define HG_LIST_HEAD_INIT(struct_head_name, var_name) \ +#define HG_LIST_HEAD_INIT(struct_head_name, var_name) \ struct struct_head_name var_name = HG_LIST_HEAD_INITIALIZER(var_name) -#define HG_LIST_HEAD_DECL(struct_head_name, struct_entry_name) \ - struct struct_head_name { \ - struct struct_entry_name *head; \ +#define HG_LIST_HEAD_DECL(struct_head_name, struct_entry_name) \ + struct struct_head_name { \ + struct struct_entry_name *head; \ } -#define HG_LIST_HEAD(struct_entry_name) \ - struct { \ - struct struct_entry_name *head; \ +#define HG_LIST_HEAD(struct_entry_name) \ + struct { \ + struct struct_entry_name *head; \ } -#define HG_LIST_ENTRY(struct_entry_name) \ - struct { \ - struct struct_entry_name *next; \ - struct struct_entry_name **prev; \ +#define HG_LIST_ENTRY(struct_entry_name) \ + struct { \ + struct struct_entry_name *next; \ + struct struct_entry_name **prev; \ } -#define HG_LIST_INIT(head_ptr) do { \ - (head_ptr)->head = NULL; \ -} while (/*CONSTCOND*/0) +#define HG_LIST_INIT(head_ptr) \ + do { \ + (head_ptr)->head = NULL; \ + } while (/*CONSTCOND*/ 0) -#define HG_LIST_IS_EMPTY(head_ptr) \ - ((head_ptr)->head == NULL) +#define HG_LIST_IS_EMPTY(head_ptr) ((head_ptr)->head == NULL) -#define HG_LIST_FIRST(head_ptr) \ - ((head_ptr)->head) +#define HG_LIST_FIRST(head_ptr) ((head_ptr)->head) -#define HG_LIST_NEXT(entry_ptr, entry_field_name) \ +#define HG_LIST_NEXT(entry_ptr, entry_field_name) \ ((entry_ptr)->entry_field_name.next) -#define HG_LIST_INSERT_AFTER(list_entry_ptr, entry_ptr, entry_field_name) do { \ - if (((entry_ptr)->entry_field_name.next = \ - (list_entry_ptr)->entry_field_name.next) != NULL) \ - (list_entry_ptr)->entry_field_name.next->entry_field_name.prev = \ - &(entry_ptr)->entry_field_name.next; \ - (list_entry_ptr)->entry_field_name.next = (entry_ptr); \ - (entry_ptr)->entry_field_name.prev = \ - &(list_entry_ptr)->entry_field_name.next; \ -} while (/*CONSTCOND*/0) - -#define HG_LIST_INSERT_BEFORE(list_entry_ptr, entry_ptr, entry_field_name) do { \ - (entry_ptr)->entry_field_name.prev = \ - (list_entry_ptr)->entry_field_name.prev; \ - (entry_ptr)->entry_field_name.next = (list_entry_ptr); \ - *(list_entry_ptr)->entry_field_name.prev = (entry_ptr); \ - (list_entry_ptr)->entry_field_name.prev = \ - &(entry_ptr)->entry_field_name.next; \ -} while (/*CONSTCOND*/0) - -#define HG_LIST_INSERT_HEAD(head_ptr, entry_ptr, entry_field_name) do { \ - if (((entry_ptr)->entry_field_name.next = (head_ptr)->head) != NULL) \ - (head_ptr)->head->entry_field_name.prev = \ - &(entry_ptr)->entry_field_name.next; \ - (head_ptr)->head = (entry_ptr); \ - (entry_ptr)->entry_field_name.prev = &(head_ptr)->head; \ -} while (/*CONSTCOND*/0) +#define HG_LIST_INSERT_AFTER(list_entry_ptr, entry_ptr, entry_field_name) \ + do { \ + if (((entry_ptr)->entry_field_name.next = \ + (list_entry_ptr)->entry_field_name.next) != NULL) \ + (list_entry_ptr)->entry_field_name.next->entry_field_name.prev = \ + &(entry_ptr)->entry_field_name.next; \ + (list_entry_ptr)->entry_field_name.next = (entry_ptr); \ + (entry_ptr)->entry_field_name.prev = \ + &(list_entry_ptr)->entry_field_name.next; \ + } while (/*CONSTCOND*/ 0) + +#define HG_LIST_INSERT_BEFORE(list_entry_ptr, entry_ptr, entry_field_name) \ + do { \ + (entry_ptr)->entry_field_name.prev = \ + (list_entry_ptr)->entry_field_name.prev; \ + (entry_ptr)->entry_field_name.next = (list_entry_ptr); \ + *(list_entry_ptr)->entry_field_name.prev = (entry_ptr); \ + (list_entry_ptr)->entry_field_name.prev = \ + &(entry_ptr)->entry_field_name.next; \ + } while (/*CONSTCOND*/ 0) + +#define HG_LIST_INSERT_HEAD(head_ptr, entry_ptr, entry_field_name) \ + do { \ + if (((entry_ptr)->entry_field_name.next = (head_ptr)->head) != NULL) \ + (head_ptr)->head->entry_field_name.prev = \ + &(entry_ptr)->entry_field_name.next; \ + (head_ptr)->head = (entry_ptr); \ + (entry_ptr)->entry_field_name.prev = &(head_ptr)->head; \ + } while (/*CONSTCOND*/ 0) /* TODO would be nice to not have any condition */ -#define HG_LIST_REMOVE(entry_ptr, entry_field_name) do { \ - if ((entry_ptr)->entry_field_name.next != NULL) \ - (entry_ptr)->entry_field_name.next->entry_field_name.prev = \ - (entry_ptr)->entry_field_name.prev; \ - *(entry_ptr)->entry_field_name.prev = (entry_ptr)->entry_field_name.next; \ -} while (/*CONSTCOND*/0) - -#define HG_LIST_FOREACH(var, head_ptr, entry_field_name) \ - for ((var) = ((head_ptr)->head); \ - (var); \ - (var) = ((var)->entry_field_name.next)) +#define HG_LIST_REMOVE(entry_ptr, entry_field_name) \ + do { \ + if ((entry_ptr)->entry_field_name.next != NULL) \ + (entry_ptr)->entry_field_name.next->entry_field_name.prev = \ + (entry_ptr)->entry_field_name.prev; \ + *(entry_ptr)->entry_field_name.prev = \ + (entry_ptr)->entry_field_name.next; \ + } while (/*CONSTCOND*/ 0) + +#define HG_LIST_FOREACH(var, head_ptr, entry_field_name) \ + for ((var) = ((head_ptr)->head); (var); \ + (var) = ((var)->entry_field_name.next)) #endif /* MERCURY_LIST_H */ diff --git a/src/util/mercury_log.c b/src/util/mercury_log.c index e17e9737..84c468a0 100644 --- a/src/util/mercury_log.c +++ b/src/util/mercury_log.c @@ -16,7 +16,20 @@ /* Local Macros */ /****************/ -#define HG_UTIL_LOG_MAX_BUF 256 +#define HG_LOG_MAX_BUF 256 + +#ifdef HG_UTIL_HAS_LOG_COLOR +# define HG_LOG_ESC "\033" +# define HG_LOG_RESET HG_LOG_ESC "[0m" +# define HG_LOG_REG HG_LOG_ESC "[0;" +# define HG_LOG_BOLD HG_LOG_ESC "[1;" +# define HG_LOG_RED "31m" +# define HG_LOG_GREEN "32m" +# define HG_LOG_YELLOW "33m" +# define HG_LOG_BLUE "34m" +# define HG_LOG_MAGENTA "35m" +# define HG_LOG_CYAN "36m" +#endif /*******************/ /* Local Variables */ @@ -60,22 +73,33 @@ void hg_log_write(unsigned int log_type, const char *module, const char *file, unsigned int line, const char *func, const char *format, ...) { - char buf[HG_UTIL_LOG_MAX_BUF]; - int desc_len; + char buf[HG_LOG_MAX_BUF]; FILE *stream = NULL; const char *msg_type = NULL; +#ifdef HG_UTIL_HAS_LOG_COLOR + const char *color = ""; +#endif va_list ap; switch (log_type) { case HG_LOG_TYPE_DEBUG: +#ifdef HG_UTIL_HAS_LOG_COLOR + color = HG_LOG_BLUE; +#endif stream = hg_log_stream_debug_g ? hg_log_stream_debug_g : stdout; msg_type = "Debug"; break; case HG_LOG_TYPE_WARNING: +#ifdef HG_UTIL_HAS_LOG_COLOR + color = HG_LOG_MAGENTA; +#endif stream = hg_log_stream_warning_g ? hg_log_stream_warning_g : stdout; msg_type = "Warning"; break; case HG_LOG_TYPE_ERROR: +#ifdef HG_UTIL_HAS_LOG_COLOR + color = HG_LOG_RED; +#endif stream = hg_log_stream_error_g ? hg_log_stream_error_g : stderr; msg_type = "Error"; break; @@ -84,17 +108,21 @@ hg_log_write(unsigned int log_type, const char *module, const char *file, }; va_start(ap, format); - desc_len = vsnprintf(buf, HG_UTIL_LOG_MAX_BUF, format, ap); -#ifdef HG_UTIL_HAS_VERBOSE_ERROR - if (desc_len > HG_UTIL_LOG_MAX_BUF) - /* Truncated */ - fprintf(stderr, "Warning, log message truncated\n"); -#else - (void) desc_len; -#endif + vsnprintf(buf, HG_LOG_MAX_BUF, format, ap); va_end(ap); - /* Print using logging function */ - hg_log_func_g(stream, "# %s -- %s -- %s:%d\n" - " # %s(): %s\n", module, msg_type, file, line, func, buf); +/* Print using logging function */ +#ifdef HG_UTIL_HAS_LOG_COLOR + hg_log_func_g(stream, + "# %s%s[%s -- %s%s%s%s%s -- %s:%d]%s\n" + "## %s%s%s()%s: %s\n", + HG_LOG_REG, color, module, HG_LOG_BOLD, color, msg_type, HG_LOG_REG, + color, file, line, HG_LOG_RESET, HG_LOG_REG, HG_LOG_YELLOW, func, + HG_LOG_RESET, buf); +#else + hg_log_func_g(stream, + "# %s -- %s -- %s:%d\n" + " # %s(): %s\n", + module, msg_type, file, line, func, buf); +#endif } diff --git a/src/util/mercury_log.h b/src/util/mercury_log.h index d6eb11eb..9ee48f20 100644 --- a/src/util/mercury_log.h +++ b/src/util/mercury_log.h @@ -21,28 +21,31 @@ #define HG_LOG_TYPE_ERROR 0x04 /* For compatibility */ -#if defined(__STDC_VERSION__) && (__STDC_VERSION__ < 199901L) - #if defined(__GNUC__) && (__GNUC__ >= 2) - #define __func__ __FUNCTION__ - #else - #define __func__ "" - #endif +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ < 199901L) +# if defined(__GNUC__) && (__GNUC__ >= 2) +# define __func__ __FUNCTION__ +# else +# define __func__ "" +# endif #elif defined(_WIN32) - #define __func__ __FUNCTION__ +# define __func__ __FUNCTION__ #endif -#define HG_LOG_WRITE_ERROR(HG_LOG_MODULE_NAME, ...) do { \ - hg_log_write(HG_LOG_TYPE_ERROR, HG_LOG_MODULE_NAME, __FILE__, \ - __LINE__, __func__, __VA_ARGS__); \ -} while (0) -#define HG_LOG_WRITE_DEBUG(HG_LOG_MODULE_NAME, ...) do { \ - hg_log_write(HG_LOG_TYPE_DEBUG, HG_LOG_MODULE_NAME, __FILE__, \ - __LINE__, __func__, __VA_ARGS__); \ -} while (0) -#define HG_LOG_WRITE_WARNING(HG_LOG_MODULE_NAME, ...) do { \ - hg_log_write(HG_LOG_TYPE_WARNING, HG_LOG_MODULE_NAME, __FILE__, \ - __LINE__, __func__, __VA_ARGS__); \ -} while (0) +#define HG_LOG_WRITE_ERROR(HG_LOG_MODULE_NAME, ...) \ + do { \ + hg_log_write(HG_LOG_TYPE_ERROR, HG_LOG_MODULE_NAME, __FILE__, \ + __LINE__, __func__, __VA_ARGS__); \ + } while (0) +#define HG_LOG_WRITE_DEBUG(HG_LOG_MODULE_NAME, ...) \ + do { \ + hg_log_write(HG_LOG_TYPE_DEBUG, HG_LOG_MODULE_NAME, __FILE__, \ + __LINE__, __func__, __VA_ARGS__); \ + } while (0) +#define HG_LOG_WRITE_WARNING(HG_LOG_MODULE_NAME, ...) \ + do { \ + hg_log_write(HG_LOG_TYPE_WARNING, HG_LOG_MODULE_NAME, __FILE__, \ + __LINE__, __func__, __VA_ARGS__); \ + } while (0) #ifdef __cplusplus extern "C" { @@ -53,7 +56,7 @@ extern "C" { * * \param log_func [IN] pointer to function */ -HG_UTIL_EXPORT void +HG_UTIL_PUBLIC void hg_log_set_func(int (*log_func)(FILE *stream, const char *format, ...)); /** @@ -61,7 +64,7 @@ hg_log_set_func(int (*log_func)(FILE *stream, const char *format, ...)); * * \param stream [IN/OUT] pointer to stream */ -HG_UTIL_EXPORT void +HG_UTIL_PUBLIC void hg_log_set_stream_debug(FILE *stream); /** @@ -69,7 +72,7 @@ hg_log_set_stream_debug(FILE *stream); * * \param stream [IN/OUT] pointer to stream */ -HG_UTIL_EXPORT void +HG_UTIL_PUBLIC void hg_log_set_stream_warning(FILE *stream); /** @@ -77,7 +80,7 @@ hg_log_set_stream_warning(FILE *stream); * * \param stream [IN/OUT] pointer to stream */ -HG_UTIL_EXPORT void +HG_UTIL_PUBLIC void hg_log_set_stream_error(FILE *stream); /** @@ -90,7 +93,7 @@ hg_log_set_stream_error(FILE *stream); * \param func [IN] function name * \param format [IN] string format */ -HG_UTIL_EXPORT void +HG_UTIL_PUBLIC void hg_log_write(unsigned int log_type, const char *module, const char *file, unsigned int line, const char *func, const char *format, ...); diff --git a/src/util/mercury_mem.c b/src/util/mercury_mem.c index eb481876..59a0acb9 100644 --- a/src/util/mercury_mem.c +++ b/src/util/mercury_mem.c @@ -9,18 +9,19 @@ */ #include "mercury_mem.h" + #include "mercury_util_error.h" #ifdef _WIN32 -# include +# include #else -# include -# include -# include -# include /* For mode constants */ -# include /* For O_* constants */ -# include -# include +# include +# include /* For O_* constants */ +# include +# include +# include /* For mode constants */ +# include +# include #endif #include @@ -32,7 +33,7 @@ hg_mem_get_page_size(void) #ifdef _WIN32 SYSTEM_INFO system_info; - GetSystemInfo (&system_info); + GetSystemInfo(&system_info); page_size = system_info.dwPageSize; #else page_size = sysconf(_SC_PAGE_SIZE); @@ -50,14 +51,13 @@ hg_mem_aligned_alloc(size_t alignment, size_t size) #ifdef _WIN32 mem_ptr = _aligned_malloc(size, alignment); #else -#ifdef _ISOC11_SOURCE +# ifdef _ISOC11_SOURCE mem_ptr = aligned_alloc(alignment, size); -#else - if (posix_memalign(&mem_ptr, alignment, size) != 0) { - HG_UTIL_LOG_ERROR("posix_memalign failed"); +# else + int rc = posix_memalign(&mem_ptr, alignment, size); + if (rc != 0) return NULL; - } -#endif +# endif #endif return mem_ptr; @@ -79,88 +79,71 @@ void * hg_mem_shm_map(const char *name, size_t size, hg_util_bool_t create) { void *mem_ptr = NULL; - int ret = HG_UTIL_SUCCESS; #ifdef _WIN32 HANDLE fd = INVALID_HANDLE_VALUE; LARGE_INTEGER large = {.QuadPart = size}; DWORD access = FILE_MAP_READ | FILE_MAP_WRITE; + BOOL rc; if (create) { fd = CreateFileMappingA(INVALID_HANDLE_VALUE, 0, PAGE_READWRITE, large.HighPart, large.LowPart, name); - if (!fd) { - HG_UTIL_LOG_ERROR("CreateFileMappingA() failed"); - ret = HG_UTIL_FAIL; - goto done; - } + HG_UTIL_CHECK_ERROR_NORET(!fd, error, "CreateFileMappingA() failed"); } else { fd = OpenFileMappingA(access, FALSE, name); - if (!fd) { - HG_UTIL_LOG_ERROR("OpenFileMappingA() failed"); - ret = HG_UTIL_FAIL; - goto done; - } + HG_UTIL_CHECK_ERROR_NORET(!fd, error, "OpenFileMappingA() failed"); } mem_ptr = MapViewOfFile(fd, access, 0, 0, size); - if (!mem_ptr) { - HG_UTIL_LOG_ERROR("MapViewOfFile() failed"); - ret = HG_UTIL_FAIL; - goto done; - } + HG_UTIL_CHECK_ERROR_NORET(!mem_ptr, error, "MapViewOfFile() failed"); /* The handle can be closed without affecting the memory mapping */ - CloseHandle(fd); + rc = CloseHandle(fd); + HG_UTIL_CHECK_ERROR_NORET(!rc, error, "CloseHandle() failed"); #else int fd = 0; int flags = O_RDWR | (create ? O_CREAT : 0); struct stat shm_stat; + int rc; fd = shm_open(name, flags, S_IRUSR | S_IWUSR); - if (fd < 0) { - HG_UTIL_LOG_ERROR("shm_open() failed (%s)", strerror(errno)); - ret = HG_UTIL_FAIL; - goto done; - } + HG_UTIL_CHECK_ERROR_NORET( + fd < 0, error, "shm_open() failed (%s)", strerror(errno)); - if (fstat(fd, &shm_stat)) { - HG_UTIL_LOG_ERROR("fstat() failed (%s)", strerror(errno)); - ret = HG_UTIL_FAIL; - goto done; - } - - if (shm_stat.st_size == 0) { - if (ftruncate(fd, (off_t) size) < 0) { - HG_UTIL_LOG_ERROR("ftruncate() failed (%s)", strerror(errno)); - ret = HG_UTIL_FAIL; - goto done; - } - } else if (shm_stat.st_size < (off_t) size) { - HG_UTIL_LOG_ERROR("shm file size too small"); - ret = HG_UTIL_FAIL; - goto done; - } + rc = fstat(fd, &shm_stat); + HG_UTIL_CHECK_ERROR_NORET( + rc != 0, error, "fstat() failed (%s)", strerror(errno)); + + if (shm_stat.st_size == 0) { + rc = ftruncate(fd, (off_t) size); + HG_UTIL_CHECK_ERROR_NORET( + rc != 0, error, "ftruncate() failed (%s)", strerror(errno)); + } else + HG_UTIL_CHECK_ERROR_NORET( + shm_stat.st_size < (off_t) size, error, "shm file size too small"); mem_ptr = mmap(NULL, size, PROT_WRITE | PROT_READ, MAP_SHARED, fd, 0); - if (mem_ptr == MAP_FAILED) { - HG_UTIL_LOG_ERROR("mmap() failed (%s)", strerror(errno)); - ret = HG_UTIL_FAIL; - goto done; - } + HG_UTIL_CHECK_ERROR_NORET( + mem_ptr == MAP_FAILED, error, "mmap() failed (%s)", strerror(errno)); /* The file descriptor can be closed without affecting the memory mapping */ - if (close(fd) == -1) { - HG_UTIL_LOG_ERROR("close() failed (%s)", strerror(errno)); - ret = HG_UTIL_FAIL; - goto done; - } + rc = close(fd); + HG_UTIL_CHECK_ERROR_NORET( + rc != 0, error, "close() failed (%s)", strerror(errno)); #endif -done: - if (ret != HG_UTIL_SUCCESS) { - /* TODO free buf */ - } return mem_ptr; + +error: +#ifdef _WIN32 + if (fd) + CloseHandle(fd); +#else + if (fd > 0) + close(fd); +#endif + + return NULL; } /*---------------------------------------------------------------------------*/ @@ -170,21 +153,22 @@ hg_mem_shm_unmap(const char *name, void *mem_ptr, size_t size) int ret = HG_UTIL_SUCCESS; #ifdef _WIN32 - if (mem_ptr) - UnmapViewOfFile(mem_ptr); -// if (fd) -// CloseHandle(fd); + if (mem_ptr) { + BOOL rc = UnmapViewOfFile(mem_ptr); + HG_UTIL_CHECK_ERROR( + !rc, done, ret, HG_UTIL_FAIL, "UnmapViewOfFile() failed"); + } #else - if (mem_ptr && mem_ptr != MAP_FAILED && munmap(mem_ptr, size) == -1) { - HG_UTIL_LOG_ERROR("munmap() failed (%s)", strerror(errno)); - ret = HG_UTIL_FAIL; - goto done; + if (mem_ptr && mem_ptr != MAP_FAILED) { + int rc = munmap(mem_ptr, size); + HG_UTIL_CHECK_ERROR(rc != 0, done, ret, HG_UTIL_FAIL, + "munmap() failed (%s)", strerror(errno)); } - if (name && shm_unlink(name) == -1) { - HG_UTIL_LOG_ERROR("shm_unlink() failed (%s)", strerror(errno)); - ret = HG_UTIL_FAIL; - goto done; + if (name) { + int rc = shm_unlink(name); + HG_UTIL_CHECK_ERROR(rc != 0, done, ret, HG_UTIL_FAIL, + "shm_unlink() failed (%s)", strerror(errno)); } #endif diff --git a/src/util/mercury_mem.h b/src/util/mercury_mem.h index b416bb03..c1ff53df 100644 --- a/src/util/mercury_mem.h +++ b/src/util/mercury_mem.h @@ -13,9 +13,20 @@ #include "mercury_util_config.h" -/** - * Purpose: memory related utility functions. - */ +/*************************************/ +/* Public Type and Struct Definition */ +/*************************************/ + +/*****************/ +/* Public Macros */ +/*****************/ + +#define HG_MEM_CACHE_LINE_SIZE 64 +#define HG_MEM_PAGE_SIZE 4096 + +/*********************/ +/* Public Prototypes */ +/*********************/ #ifdef __cplusplus extern "C" { @@ -26,7 +37,7 @@ extern "C" { * * \return page size on success or negative on failure */ -HG_UTIL_EXPORT long +HG_UTIL_PUBLIC long hg_mem_get_page_size(void); /** @@ -39,7 +50,7 @@ hg_mem_get_page_size(void); * * \return a pointer to the allocated memory, or NULL in case of failure */ -HG_UTIL_EXPORT void * +HG_UTIL_PUBLIC void * hg_mem_aligned_alloc(size_t alignment, size_t size); /** @@ -47,7 +58,7 @@ hg_mem_aligned_alloc(size_t alignment, size_t size); * * \param mem_ptr [IN] pointer to allocated memory */ -HG_UTIL_EXPORT void +HG_UTIL_PUBLIC void hg_mem_aligned_free(void *mem_ptr); /** @@ -59,7 +70,7 @@ hg_mem_aligned_free(void *mem_ptr); * * \return a pointer to the mapped memory region, or NULL in case of failure */ -HG_UTIL_EXPORT void * +HG_UTIL_PUBLIC void * hg_mem_shm_map(const char *name, size_t size, hg_util_bool_t create); /** @@ -71,7 +82,7 @@ hg_mem_shm_map(const char *name, size_t size, hg_util_bool_t create); * * \return non-negative on success, or negative in case of failure */ -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_mem_shm_unmap(const char *name, void *mem_ptr, size_t size); #ifdef __cplusplus diff --git a/src/util/mercury_poll.c b/src/util/mercury_poll.c index aac409f2..57f7e3e7 100644 --- a/src/util/mercury_poll.c +++ b/src/util/mercury_poll.c @@ -9,31 +9,44 @@ */ #include "mercury_poll.h" +#include "mercury_atomic.h" +#include "mercury_event.h" #include "mercury_list.h" -#include "mercury_util_error.h" #include "mercury_thread_spin.h" -#include "mercury_atomic.h" +#include "mercury_util_error.h" #include -#define HG_POLL_MAX_EVENTS 64 /* TODO Make this configurable */ - #if defined(_WIN32) /* TODO */ #else -#include -#include -#include -#if defined(HG_UTIL_HAS_SYSEPOLL_H) -#include -#elif defined(HG_UTIL_HAS_SYSEVENT_H) -#include -#include -#else -#include -#endif +# include +# include +# include +# if defined(HG_UTIL_HAS_SYSEPOLL_H) +# include +# elif defined(HG_UTIL_HAS_SYSEVENT_H) +# include +# include +# else +# include +# endif #endif /* defined(_WIN32) */ +/****************/ +/* Local Macros */ +/****************/ + +#define HG_POLL_MAX_EVENTS 1024 + +#ifndef MIN +# define MIN(a, b) (((a) < (b)) ? (a) : (b)) +#endif + +/************************************/ +/* Local Type and Struct Definition */ +/************************************/ + struct hg_poll_data { #if defined(HG_UTIL_HAS_SYSEPOLL_H) int fd; @@ -52,29 +65,27 @@ struct hg_poll_set { hg_atomic_int32_t nfds; hg_poll_try_wait_cb_t try_wait_cb; void *try_wait_arg; -#if defined(HG_UTIL_HAS_SYSEPOLL_H) || defined(HG_UTIL_HAS_SYSEVENT_H) - /* Nothing */ -#else - struct pollfd *poll_fds; -#endif HG_LIST_HEAD(hg_poll_data) poll_data_list; hg_thread_spin_t poll_data_list_lock; }; +/********************/ +/* Local Prototypes */ +/********************/ + +/*******************/ +/* Local Variables */ +/*******************/ + /*---------------------------------------------------------------------------*/ hg_poll_set_t * hg_poll_create(void) { struct hg_poll_set *hg_poll_set = NULL; -#if defined(HG_UTIL_HAS_SYSEPOLL_H) || defined(HG_UTIL_HAS_SYSEVENT_H) - int ret = 0; -#endif hg_poll_set = malloc(sizeof(struct hg_poll_set)); - if (!hg_poll_set) { - HG_UTIL_LOG_ERROR("malloc() failed (%s)"); - goto done; - } + HG_UTIL_CHECK_ERROR_NORET( + hg_poll_set == NULL, error, "malloc() failed (%s)"); #if defined(_WIN32) /* TODO */ #else @@ -82,37 +93,30 @@ hg_poll_create(void) hg_thread_spin_init(&hg_poll_set->poll_data_list_lock); hg_atomic_init32(&hg_poll_set->nfds, 0); hg_poll_set->try_wait_cb = NULL; -#if defined(HG_UTIL_HAS_SYSEPOLL_H) - ret = epoll_create1(0); - if (ret == -1) { - HG_UTIL_LOG_ERROR("epoll_create1() failed (%s)", strerror(errno)); - free(hg_poll_set); - hg_poll_set = NULL; - goto done; - } - hg_poll_set->fd = ret; -#elif defined(HG_UTIL_HAS_SYSEVENT_H) - ret = kqueue(); - if (ret == -1) { - HG_UTIL_LOG_ERROR("kqueue() failed (%s)", strerror(errno)); - free(hg_poll_set); - hg_poll_set = NULL; - goto done; - } - hg_poll_set->fd = ret; -#else - hg_poll_set->poll_fds = malloc(sizeof(int) * HG_POLL_MAX_EVENTS); - if (!hg_poll_set->poll_fds) { - HG_UTIL_LOG_ERROR("malloc() failed (%s)"); - free(hg_poll_set); - hg_poll_set = NULL; - goto done; - } -#endif + +# if defined(HG_UTIL_HAS_SYSEPOLL_H) + hg_poll_set->fd = epoll_create1(0); + HG_UTIL_CHECK_ERROR_NORET(hg_poll_set->fd == -1, error, + "epoll_create1() failed (%s)", strerror(errno)); +# elif defined(HG_UTIL_HAS_SYSEVENT_H) + hg_poll_set->fd = kqueue(); + HG_UTIL_CHECK_ERROR_NORET( + hg_poll_set->fd == -1, error, "kqueue() failed (%s)", strerror(errno)); +# else + hg_poll_set->fd = hg_event_create(); + HG_UTIL_CHECK_ERROR_NORET(hg_poll_set->fd == -1, error, + "hg_event_create() failed (%s)", strerror(errno)); +# endif #endif /* defined(_WIN32) */ -done: return hg_poll_set; + +error: + if (hg_poll_set) { + hg_thread_spin_destroy(&hg_poll_set->poll_data_list_lock); + free(hg_poll_set); + } + return NULL; } /*---------------------------------------------------------------------------*/ @@ -120,30 +124,31 @@ int hg_poll_destroy(hg_poll_set_t *poll_set) { int ret = HG_UTIL_SUCCESS; + int rc; if (!poll_set) goto done; #if defined(_WIN32) - /* TODO */ + /* TODO */ #else - if (hg_atomic_get32(&poll_set->nfds)) { - HG_UTIL_LOG_ERROR("Poll set non empty"); - ret = HG_UTIL_FAIL; - goto done; - } -#if defined(HG_UTIL_HAS_SYSEPOLL_H) || defined(HG_UTIL_HAS_SYSEVENT_H) + HG_UTIL_CHECK_ERROR(hg_atomic_get32(&poll_set->nfds), done, ret, + HG_UTIL_FAIL, "Poll set non empty"); + +# if defined(HG_UTIL_HAS_SYSEPOLL_H) || defined(HG_UTIL_HAS_SYSEVENT_H) /* Close poll descriptor */ - if (close(poll_set->fd) == -1) { - HG_UTIL_LOG_ERROR("close() failed (%s)", strerror(errno)); - ret = HG_UTIL_FAIL; - goto done; - } -#else - free(poll_set->poll_fds); -#endif + rc = close(poll_set->fd); + HG_UTIL_CHECK_ERROR(rc == -1, done, ret, HG_UTIL_FAIL, + "close() failed (%s)", strerror(errno)); +# else + rc = hg_event_destroy(poll_set->fd); + HG_UTIL_CHECK_ERROR(rc == HG_UTIL_FAIL, done, ret, HG_UTIL_FAIL, + "hg_event_destroy() failed (%s)", strerror(errno)); +# endif + hg_thread_spin_destroy(&poll_set->poll_data_list_lock); #endif /* defined(_WIN32) */ + free(poll_set); done: @@ -154,19 +159,14 @@ hg_poll_destroy(hg_poll_set_t *poll_set) int hg_poll_get_fd(hg_poll_set_t *poll_set) { - int fd = 0; + int fd = -1; + + HG_UTIL_CHECK_ERROR_NORET(!poll_set, done, "NULL poll set"); - if (!poll_set) { - HG_UTIL_LOG_ERROR("NULL poll set"); - fd = HG_UTIL_FAIL; - goto done; - } #if defined(_WIN32) /* TODO */ -#elif defined(HG_UTIL_HAS_SYSEPOLL_H) || defined(HG_UTIL_HAS_SYSEVENT_H) - fd = poll_set->fd; #else - /* TODO */ + fd = poll_set->fd; #endif done: @@ -175,16 +175,12 @@ hg_poll_get_fd(hg_poll_set_t *poll_set) /*---------------------------------------------------------------------------*/ int -hg_poll_set_try_wait(hg_poll_set_t *poll_set, hg_poll_try_wait_cb_t try_wait_cb, - void *arg) +hg_poll_set_try_wait( + hg_poll_set_t *poll_set, hg_poll_try_wait_cb_t try_wait_cb, void *arg) { int ret = HG_UTIL_SUCCESS; - if (!poll_set) { - HG_UTIL_LOG_ERROR("NULL poll set"); - ret = HG_UTIL_FAIL; - goto done; - } + HG_UTIL_CHECK_ERROR(!poll_set, done, ret, HG_UTIL_FAIL, "NULL poll set"); poll_set->try_wait_cb = try_wait_cb; poll_set->try_wait_arg = arg; @@ -201,18 +197,12 @@ hg_poll_add(hg_poll_set_t *poll_set, int fd, unsigned int flags, struct hg_poll_data *hg_poll_data = NULL; int ret = HG_UTIL_SUCCESS; - if (!poll_set) { - HG_UTIL_LOG_ERROR("NULL poll set"); - ret = HG_UTIL_FAIL; - goto done; - } + HG_UTIL_CHECK_ERROR(!poll_set, done, ret, HG_UTIL_FAIL, "NULL poll set"); /* Allocate poll data that can hold user data and callback */ hg_poll_data = malloc(sizeof(struct hg_poll_data)); - if (!hg_poll_data) { - HG_UTIL_LOG_ERROR("malloc() failed (%s)"); - goto done; - } + HG_UTIL_CHECK_ERROR( + !hg_poll_data, done, ret, HG_UTIL_FAIL, "malloc() failed (%s)"); memset(hg_poll_data, 0, sizeof(struct hg_poll_data)); hg_poll_data->poll_cb = poll_cb; hg_poll_data->poll_arg = poll_arg; @@ -223,6 +213,7 @@ hg_poll_add(hg_poll_set_t *poll_set, int fd, unsigned int flags, #elif defined(HG_UTIL_HAS_SYSEPOLL_H) struct epoll_event ev; uint32_t poll_flags; + int rc; /* Translate flags */ switch (flags) { @@ -233,23 +224,20 @@ hg_poll_add(hg_poll_set_t *poll_set, int fd, unsigned int flags, poll_flags = EPOLLOUT; break; default: - HG_UTIL_LOG_ERROR("Invalid flag"); - ret = HG_UTIL_FAIL; - goto done; + HG_UTIL_GOTO_ERROR(error, ret, HG_UTIL_FAIL, "Invalid flag"); } hg_poll_data->fd = fd; ev.events = poll_flags; ev.data.ptr = hg_poll_data; - if (epoll_ctl(poll_set->fd, EPOLL_CTL_ADD, fd, &ev) == -1) { - HG_UTIL_LOG_ERROR("epoll_ctl() failed (%s)", strerror(errno)); - ret = HG_UTIL_FAIL; - goto done; - } + rc = epoll_ctl(poll_set->fd, EPOLL_CTL_ADD, fd, &ev); + HG_UTIL_CHECK_ERROR(rc != 0, error, ret, HG_UTIL_FAIL, + "epoll_ctl() failed (%s)", strerror(errno)); #elif defined(HG_UTIL_HAS_SYSEVENT_H) struct timespec timeout = {0, 0}; int16_t poll_flags; + int rc; /* Translate flags */ switch (flags) { @@ -260,18 +248,15 @@ hg_poll_add(hg_poll_set_t *poll_set, int fd, unsigned int flags, poll_flags = EVFILT_WRITE; break; default: - HG_UTIL_LOG_ERROR("Invalid flag"); - ret = HG_UTIL_FAIL; - goto done; + HG_UTIL_GOTO_ERROR(error, ret, HG_UTIL_FAIL, "Invalid flag"); } - EV_SET(&hg_poll_data->kev, (uintptr_t) fd, poll_flags, EV_ADD, 0, 0, hg_poll_data); + EV_SET(&hg_poll_data->kev, (uintptr_t) fd, poll_flags, EV_ADD, 0, 0, + hg_poll_data); - if (kevent(poll_set->fd, &hg_poll_data->kev, 1, NULL, 0, &timeout) == -1) { - HG_UTIL_LOG_ERROR("kevent() failed (%s)", strerror(errno)); - ret = HG_UTIL_FAIL; - goto done; - } + rc = kevent(poll_set->fd, &hg_poll_data->kev, 1, NULL, 0, &timeout); + HG_UTIL_CHECK_ERROR(rc == -1, error, ret, HG_UTIL_FAIL, + "kevent() failed (%s)", strerror(errno)); #else short int poll_flags; @@ -284,35 +269,27 @@ hg_poll_add(hg_poll_set_t *poll_set, int fd, unsigned int flags, poll_flags = POLLOUT; break; default: - HG_UTIL_LOG_ERROR("Invalid flag"); - ret = HG_UTIL_FAIL; - goto done; + HG_UTIL_GOTO_ERROR(error, ret, HG_UTIL_FAIL, "Invalid flag"); } hg_poll_data->pollfd.fd = fd; hg_poll_data->pollfd.events = poll_flags; hg_poll_data->pollfd.revents = 0; - - /* TODO limit on number of fds for now but could malloc/reallocate */ - if (poll_set->nfds + 1 > HG_POLL_MAX_EVENTS) { - HG_UTIL_LOG_ERROR("Exceeding number of pollable file descriptors"); - ret = HG_UTIL_FAIL; - free(hg_poll_data); - goto done; - } - - poll_set->poll_fds[poll_set->nfds] = hg_poll_data->pollfd; #endif /* defined(_WIN32) */ } + hg_atomic_incr32(&poll_set->nfds); + hg_thread_spin_lock(&poll_set->poll_data_list_lock); HG_LIST_INSERT_HEAD(&poll_set->poll_data_list, hg_poll_data, entry); hg_thread_spin_unlock(&poll_set->poll_data_list_lock); - hg_atomic_incr32(&poll_set->nfds); done: - if (ret != HG_UTIL_SUCCESS) - free(hg_poll_data); return ret; + +error: + free(hg_poll_data); + + return HG_UTIL_FAIL; } /*---------------------------------------------------------------------------*/ @@ -323,162 +300,126 @@ hg_poll_remove(hg_poll_set_t *poll_set, int fd) hg_util_bool_t found = HG_UTIL_FALSE; int ret = HG_UTIL_SUCCESS; - if (!poll_set) { - HG_UTIL_LOG_ERROR("NULL poll set"); - ret = HG_UTIL_FAIL; - goto done; - } + HG_UTIL_CHECK_ERROR(!poll_set, done, ret, HG_UTIL_FAIL, "NULL poll set"); + hg_thread_spin_lock(&poll_set->poll_data_list_lock); + HG_LIST_FOREACH (hg_poll_data, &poll_set->poll_data_list, entry) { #if defined(_WIN32) - /* TODO */ + /* TODO */ #elif defined(HG_UTIL_HAS_SYSEPOLL_H) - hg_thread_spin_lock(&poll_set->poll_data_list_lock); - HG_LIST_FOREACH(hg_poll_data, &poll_set->poll_data_list, entry) { if (hg_poll_data->fd == fd) { HG_LIST_REMOVE(hg_poll_data, entry); - if ((fd > 0) - && epoll_ctl(poll_set->fd, EPOLL_CTL_DEL, fd, NULL) == -1) { - HG_UTIL_LOG_ERROR("epoll_ctl() failed (%s)", strerror(errno)); - ret = HG_UTIL_FAIL; - hg_thread_spin_unlock(&poll_set->poll_data_list_lock); - goto done; + if (fd > 0) { + int rc = epoll_ctl(poll_set->fd, EPOLL_CTL_DEL, fd, NULL); + HG_UTIL_CHECK_ERROR(rc != 0, error, ret, HG_UTIL_FAIL, + "epoll_ctl() failed (%s)", strerror(errno)); } free(hg_poll_data); found = HG_UTIL_TRUE; break; } - } - hg_thread_spin_unlock(&poll_set->poll_data_list_lock); #elif defined(HG_UTIL_HAS_SYSEVENT_H) - /* Events which are attached to file descriptors are automatically deleted - * on the last close of the descriptor. */ - hg_thread_spin_lock(&poll_set->poll_data_list_lock); - HG_LIST_FOREACH(hg_poll_data, &poll_set->poll_data_list, entry) { + /* Events which are attached to file descriptors are automatically + * deleted on the last close of the descriptor. */ if ((int) hg_poll_data->kev.ident == fd) { HG_LIST_REMOVE(hg_poll_data, entry); if (fd > 0) { struct timespec timeout = {0, 0}; - - EV_SET(&hg_poll_data->kev, (uintptr_t) fd, EVFILT_READ, EV_DELETE, 0, 0, NULL); - if (kevent(poll_set->fd, &hg_poll_data->kev, 1, NULL, 0, - &timeout) == -1) { - HG_UTIL_LOG_ERROR("kevent() failed (%s)", strerror(errno)); - ret = HG_UTIL_FAIL; - hg_thread_spin_unlock(&poll_set->poll_data_list_lock); - goto done; - } + int rc; + + EV_SET(&hg_poll_data->kev, (uintptr_t) fd, EVFILT_READ, + EV_DELETE, 0, 0, NULL); + rc = kevent( + poll_set->fd, &hg_poll_data->kev, 1, NULL, 0, &timeout); + HG_UTIL_CHECK_ERROR(rc == -1, error, ret, HG_UTIL_FAIL, + "kevent() failed (%s)", strerror(errno)); } free(hg_poll_data); found = HG_UTIL_TRUE; break; } - } - hg_thread_spin_unlock(&poll_set->poll_data_list_lock); #else - hg_thread_spin_lock(&poll_set->poll_data_list_lock); - HG_LIST_FOREACH(hg_poll_data, &poll_set->poll_data_list, entry) { if (hg_poll_data->pollfd.fd == fd) { - unsigned int i = 0; - HG_LIST_REMOVE(hg_poll_data, entry); free(hg_poll_data); found = HG_UTIL_TRUE; - - if (fd > 0) { - /* Re-order poll_events */ - HG_LIST_FOREACH(hg_poll_data, &poll_set->poll_data_list, entry) { - poll_set->poll_fds[i] = hg_poll_data->pollfd; - i++; - } - } break; } - } - hg_thread_spin_unlock(&poll_set->poll_data_list_lock); #endif - if (!found) { - HG_UTIL_LOG_ERROR("Could not find fd in poll_set"); - ret = HG_UTIL_FAIL; - goto done; } + hg_thread_spin_unlock(&poll_set->poll_data_list_lock); + + HG_UTIL_CHECK_ERROR( + !found, done, ret, HG_UTIL_FAIL, "Could not find fd in poll_set"); hg_atomic_decr32(&poll_set->nfds); done: return ret; + +#if defined(HG_UTIL_HAS_SYSEPOLL_H) || defined(HG_UTIL_HAS_SYSEVENT_H) +error: + hg_thread_spin_unlock(&poll_set->poll_data_list_lock); + + return ret; +#endif } /*---------------------------------------------------------------------------*/ int hg_poll_wait(hg_poll_set_t *poll_set, unsigned int timeout, - hg_util_bool_t *progressed) + unsigned int max_events, struct hg_poll_event *events, + unsigned int *actual_events) { - hg_util_bool_t poll_progressed = HG_UTIL_FALSE; + int max_poll_events = (int) MIN(max_events, HG_POLL_MAX_EVENTS); + int nfds = 0, i; int ret = HG_UTIL_SUCCESS; - if (!poll_set) { - HG_UTIL_LOG_ERROR("NULL poll set"); - ret = HG_UTIL_FAIL; - goto done; - } + HG_UTIL_CHECK_ERROR(!poll_set, done, ret, HG_UTIL_FAIL, "NULL poll set"); - if (timeout && (!poll_set->try_wait_cb || (poll_set->try_wait_cb - && poll_set->try_wait_cb(poll_set->try_wait_arg)))) { + if (timeout && (!poll_set->try_wait_cb || + (poll_set->try_wait_cb && + poll_set->try_wait_cb(poll_set->try_wait_arg)))) { #if defined(_WIN32) #elif defined(HG_UTIL_HAS_SYSEPOLL_H) - struct epoll_event events[HG_POLL_MAX_EVENTS]; - int nfds, i; - - nfds = epoll_wait(poll_set->fd, events, HG_POLL_MAX_EVENTS, (int) timeout); - if (nfds == -1 && errno != EINTR) { - HG_UTIL_LOG_ERROR("epoll_wait() failed (%s)", strerror(errno)); - ret = HG_UTIL_FAIL; - goto done; - } + struct epoll_event poll_events[HG_POLL_MAX_EVENTS]; + + nfds = epoll_wait( + poll_set->fd, poll_events, max_poll_events, (int) timeout); + HG_UTIL_CHECK_ERROR(nfds == -1 && errno != EINTR, done, ret, + HG_UTIL_FAIL, "epoll_wait() failed (%s)", strerror(errno)); + for (i = 0; i < nfds; ++i) { struct hg_poll_data *hg_poll_data = - (struct hg_poll_data *) events[i].data.ptr; - int error = 0; + (struct hg_poll_data *) poll_events[i].data.ptr; + int error = 0, rc; - if (!hg_poll_data) { - HG_UTIL_LOG_ERROR("NULL poll data"); - ret = HG_UTIL_FAIL; - goto done; - } + HG_UTIL_CHECK_ERROR(hg_poll_data == NULL, done, ret, HG_UTIL_FAIL, + "NULL poll data"); /* Don't change the if/else order */ - if (events[i].events & EPOLLERR) { + if (poll_events[i].events & EPOLLERR) error = EPOLLERR; - } else if (events[i].events & EPOLLHUP) { + else if (poll_events[i].events & EPOLLHUP) error = EPOLLHUP; - } else if (events[i].events & EPOLLRDHUP) { + else if (poll_events[i].events & EPOLLRDHUP) error = EPOLLRDHUP; - } - if ((events[i].events & (EPOLLIN | EPOLLOUT)) - && hg_poll_data->poll_cb) { - hg_util_bool_t poll_cb_progressed = HG_UTIL_FALSE; - int poll_ret = HG_UTIL_SUCCESS; - - poll_ret = hg_poll_data->poll_cb( - hg_poll_data->poll_arg, error, &poll_cb_progressed); - if (poll_ret != HG_UTIL_SUCCESS) { - HG_UTIL_LOG_ERROR("poll cb failed"); - ret = HG_UTIL_FAIL; - goto done; - } - poll_progressed |= poll_cb_progressed; - } else { - HG_UTIL_LOG_ERROR("Unknown error has occurred"); - ret = HG_UTIL_FAIL; - goto done; - } + HG_UTIL_CHECK_ERROR(!(poll_events[i].events & (EPOLLIN | EPOLLOUT)), + done, ret, HG_UTIL_FAIL, "Unsupported events"); + + if (!hg_poll_data->poll_cb) + continue; + + rc = hg_poll_data->poll_cb( + hg_poll_data->poll_arg, error, &events[i]); + HG_UTIL_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, HG_UTIL_FAIL, + "poll cb failed"); } #elif defined(HG_UTIL_HAS_SYSEVENT_H) - struct kevent events[HG_POLL_MAX_EVENTS]; - int nfds, i; + struct kevent poll_events[HG_POLL_MAX_EVENTS]; struct timespec timeout_spec; ldiv_t ld; @@ -487,115 +428,103 @@ hg_poll_wait(hg_poll_set_t *poll_set, unsigned int timeout, timeout_spec.tv_sec = ld.quot; timeout_spec.tv_nsec = ld.rem * 1000000L; - nfds = kevent(poll_set->fd, NULL, 0, events, HG_POLL_MAX_EVENTS, - &timeout_spec); - if (nfds == -1 && errno != EINTR) { - HG_UTIL_LOG_ERROR("kevent() failed (%s)", strerror(errno)); - ret = HG_UTIL_FAIL; - goto done; - } + nfds = kevent( + poll_set->fd, NULL, 0, poll_events, max_events, &timeout_spec); + HG_UTIL_CHECK_ERROR(nfds == -1 && errno != EINTR, done, ret, + HG_UTIL_FAIL, "kevent() failed (%s)", strerror(errno)); + for (i = 0; i < nfds; ++i) { struct hg_poll_data *hg_poll_data = - (struct hg_poll_data *) events[i].udata; - if (!hg_poll_data) { - HG_UTIL_LOG_ERROR("NULL poll data"); - ret = HG_UTIL_FAIL; - goto done; - } - if (hg_poll_data->poll_cb) { - hg_util_bool_t poll_cb_progressed = HG_UTIL_FALSE; - int poll_ret = HG_UTIL_SUCCESS; - - poll_ret = hg_poll_data->poll_cb( - hg_poll_data->poll_arg, 0, &poll_cb_progressed); - if (poll_ret != HG_UTIL_SUCCESS) { - HG_UTIL_LOG_ERROR("poll cb failed"); - ret = HG_UTIL_FAIL; - goto done; - } - poll_progressed |= poll_cb_progressed; - } + (struct hg_poll_data *) poll_events[i].udata; + int rc; + + HG_UTIL_CHECK_ERROR(hg_poll_data == NULL, done, ret, HG_UTIL_FAIL, + "NULL poll data"); + + if (!hg_poll_data->poll_cb) + continue; + + rc = hg_poll_data->poll_cb(hg_poll_data->poll_arg, 0, &events[i]); + HG_UTIL_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, HG_UTIL_FAIL, + "poll cb failed"); } #else + struct pollfd poll_events[HG_POLL_MAX_EVENTS] = {0}; + struct hg_poll_data *poll_data_events[HG_POLL_MAX_EVENTS] = {NULL}; struct hg_poll_data *hg_poll_data = NULL; - int nfds; - unsigned int i; + int nevents = 0; /* Reset revents */ - for (i = 0; i < poll_set->nfds; i++) - poll_set->poll_fds[i].revents = 0; - - nfds = poll(poll_set->poll_fds, poll_set->nfds, (int) timeout); - if (nfds == -1 && errno != EINTR) { - HG_UTIL_LOG_ERROR("poll() failed (%s)", strerror(errno)); - ret = HG_UTIL_FAIL; - goto done; + hg_thread_spin_lock(&poll_set->poll_data_list_lock); + for (hg_poll_data = HG_LIST_FIRST(&poll_set->poll_data_list); + hg_poll_data && (nevents < max_poll_events); + hg_poll_data = HG_LIST_NEXT(hg_poll_data, entry), nevents++) { + poll_events[nevents] = hg_poll_data->pollfd; + poll_data_events[nevents] = hg_poll_data; } - if (nfds > 0) { - /* An event on one of the fds has occurred. */ - for (i = 0; i < poll_set->nfds; i++) { - if (poll_set->poll_fds[i].revents & poll_set->poll_fds[i].events) { - hg_thread_spin_lock(&poll_set->poll_data_list_lock); - HG_LIST_FOREACH(hg_poll_data, &poll_set->poll_data_list, entry) { - if (hg_poll_data->pollfd.fd == poll_set->poll_fds[i].fd) { - hg_thread_spin_unlock(&poll_set->poll_data_list_lock); - break; - } - } - hg_thread_spin_unlock(&poll_set->poll_data_list_lock); - - /* TODO check POLLHUP | POLLERR | POLLNVAL */ - if (hg_poll_data->poll_cb) { - hg_util_bool_t poll_cb_progressed = HG_UTIL_FALSE; - int poll_ret = HG_UTIL_SUCCESS; - - poll_ret = hg_poll_data->poll_cb( - hg_poll_data->poll_arg, 0, &poll_progressed); - if (poll_ret != HG_UTIL_SUCCESS) { - HG_UTIL_LOG_ERROR("poll cb failed"); - ret = HG_UTIL_FAIL; - goto done; - } - poll_progressed |= poll_cb_progressed; - } - } - } + hg_thread_spin_unlock(&poll_set->poll_data_list_lock); + + nfds = poll(poll_events, nevents, (int) timeout); + HG_UTIL_CHECK_ERROR(nfds == -1 && errno != EINTR, done, ret, + HG_UTIL_FAIL, "poll() failed (%s)", strerror(errno)); + + /* An event on one of the fds has occurred. */ + for (i = 0; i < nfds; ++i) { + int rc; + + if (!(poll_events[i].revents & poll_events[i].events)) + continue; + + /* TODO check POLLHUP | POLLERR | POLLNVAL */ + if (!poll_data_events[i]->poll_cb) + continue; + + rc = poll_data_events[i]->poll_cb( + poll_data_events[i]->poll_arg, 0, &events[i]); + HG_UTIL_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, HG_UTIL_FAIL, + "poll cb failed"); + } + + if (nfds) { + /* TODO should figure where to call hg_event_get() */ + int rc = hg_event_set(poll_set->fd); + HG_UTIL_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, HG_UTIL_FAIL, + "hg_event_set() failed (%s)", strerror(errno)); } #endif } else { #ifdef _WIN32 #else + struct hg_poll_data *poll_data_events[HG_POLL_MAX_EVENTS] = {NULL}; struct hg_poll_data *hg_poll_data; + int nevents = 0; + /* Reset revents */ hg_thread_spin_lock(&poll_set->poll_data_list_lock); - HG_LIST_FOREACH(hg_poll_data, &poll_set->poll_data_list, entry) { - hg_thread_spin_unlock(&poll_set->poll_data_list_lock); - if (hg_poll_data->poll_cb) { - hg_util_bool_t poll_cb_progressed = HG_UTIL_FALSE; - int poll_ret = HG_UTIL_SUCCESS; - - poll_ret = hg_poll_data->poll_cb( - hg_poll_data->poll_arg, 0, &poll_cb_progressed); - if (poll_ret != HG_UTIL_SUCCESS) { - HG_UTIL_LOG_ERROR("poll cb failed"); - ret = HG_UTIL_FAIL; - hg_thread_spin_unlock(&poll_set->poll_data_list_lock); - goto done; - } - poll_progressed |= poll_cb_progressed; - if (poll_progressed) - break; - } - hg_thread_spin_lock(&poll_set->poll_data_list_lock); + for (hg_poll_data = HG_LIST_FIRST(&poll_set->poll_data_list); + hg_poll_data && (nevents < max_poll_events); + hg_poll_data = HG_LIST_NEXT(hg_poll_data, entry), nevents++) + poll_data_events[nevents] = hg_poll_data; + hg_thread_spin_unlock(&poll_set->poll_data_list_lock); + + nfds = nevents; + for (i = 0; i < nfds; ++i) { + int rc; + + if (!poll_data_events[i]->poll_cb) + continue; + + rc = poll_data_events[i]->poll_cb( + poll_data_events[i]->poll_arg, 0, &events[i]); + HG_UTIL_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, HG_UTIL_FAIL, + "poll cb failed"); } - if (!poll_progressed) - hg_thread_spin_unlock(&poll_set->poll_data_list_lock); #endif } - if (progressed) - *progressed = poll_progressed; + if (actual_events) + *actual_events = (unsigned int) nfds; done: return ret; diff --git a/src/util/mercury_poll.h b/src/util/mercury_poll.h index 35f76cb6..8922f378 100644 --- a/src/util/mercury_poll.h +++ b/src/util/mercury_poll.h @@ -18,8 +18,17 @@ * without entering system calls. */ +/*************************************/ +/* Public Type and Struct Definition */ +/*************************************/ + typedef struct hg_poll_set hg_poll_set_t; +struct hg_poll_event { + hg_util_bool_t progressed; /* Indicates progress */ + void *ptr; /* Pointer to user data */ +}; + /** * Callback that can be used to signal when it is safe to block on the * poll set or if blocking could hang the application. @@ -31,22 +40,30 @@ typedef struct hg_poll_set hg_poll_set_t; typedef hg_util_bool_t (*hg_poll_try_wait_cb_t)(void *arg); /** - * Polling callback, arg can be used to pass user arguments, progressed - * indicates whether progress has been done after that call returns. + * Polling callback, arg can be used to pass user arguments, event can be used + * to return user arguments back to hg_poll_wait. * * \param arg [IN] pointer to user data * \param error [IN] any error event has occurred - * \param progressed [OUT] pointer to boolean indicating progress made + * \param ptr [OUT] event data output * * \return Non-negative on success or negative on failure */ -typedef int (*hg_poll_cb_t)(void *arg, int error, hg_util_bool_t *progressed); +typedef int (*hg_poll_cb_t)(void *arg, int error, struct hg_poll_event *event); + +/*****************/ +/* Public Macros */ +/*****************/ /** * Polling events. */ -#define HG_POLLIN 0x001 /* Ready to read. */ -#define HG_POLLOUT 0x004 /* Ready to write. */ +#define HG_POLLIN 0x001 /* Ready to read. */ +#define HG_POLLOUT 0x004 /* Ready to write. */ + +/*********************/ +/* Public Prototypes */ +/*********************/ #ifdef __cplusplus extern "C" { @@ -57,7 +74,7 @@ extern "C" { * * \return Pointer to poll set or NULL in case of failure */ -HG_UTIL_EXPORT hg_poll_set_t * +HG_UTIL_PUBLIC hg_poll_set_t * hg_poll_create(void); /** @@ -67,7 +84,7 @@ hg_poll_create(void); * * \return Non-negative on success or negative on failure */ -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_poll_destroy(hg_poll_set_t *poll_set); /** @@ -77,7 +94,7 @@ hg_poll_destroy(hg_poll_set_t *poll_set); * * \return Non-negative on success or negative on failure */ -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_poll_get_fd(hg_poll_set_t *poll_set); /** @@ -91,7 +108,7 @@ hg_poll_get_fd(hg_poll_set_t *poll_set); * * \return Non-negative on success or negative on failure */ -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_poll_set_try_wait(hg_poll_set_t *poll_set, hg_poll_try_wait_cb_t try_wait_cb, void *try_wait_arg); @@ -106,7 +123,7 @@ hg_poll_set_try_wait(hg_poll_set_t *poll_set, hg_poll_try_wait_cb_t try_wait_cb, * * \return Non-negative on success or negative on failure */ -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_poll_add(hg_poll_set_t *poll_set, int fd, unsigned int flags, hg_poll_cb_t poll_cb, void *poll_cb_arg); @@ -118,7 +135,7 @@ hg_poll_add(hg_poll_set_t *poll_set, int fd, unsigned int flags, * * \return Non-negative on success or negative on failure */ -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_poll_remove(hg_poll_set_t *poll_set, int fd); /** @@ -135,9 +152,10 @@ hg_poll_remove(hg_poll_set_t *poll_set, int fd); * * \return Non-negative on success or negative on failure */ -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_poll_wait(hg_poll_set_t *poll_set, unsigned int timeout, - hg_util_bool_t *progressed); + unsigned int max_events, struct hg_poll_event events[], + unsigned int *actual_events); #ifdef __cplusplus } diff --git a/src/util/mercury_queue.h b/src/util/mercury_queue.h index 5bc21dc1..2133383b 100644 --- a/src/util/mercury_queue.h +++ b/src/util/mercury_queue.h @@ -43,75 +43,81 @@ #ifndef MERCURY_QUEUE_H #define MERCURY_QUEUE_H -#define HG_QUEUE_HEAD_INITIALIZER(name) { NULL, &(name).head } +#define HG_QUEUE_HEAD_INITIALIZER(name) \ + { \ + NULL, &(name).head \ + } -#define HG_QUEUE_HEAD_INIT(struct_head_name, var_name) \ +#define HG_QUEUE_HEAD_INIT(struct_head_name, var_name) \ struct struct_head_name var_name = HG_QUEUE_HEAD_INITIALIZER(var_name) -#define HG_QUEUE_HEAD_DECL(struct_head_name, struct_entry_name) \ - struct struct_head_name { \ - struct struct_entry_name *head; \ - struct struct_entry_name **tail; \ +#define HG_QUEUE_HEAD_DECL(struct_head_name, struct_entry_name) \ + struct struct_head_name { \ + struct struct_entry_name *head; \ + struct struct_entry_name **tail; \ } -#define HG_QUEUE_HEAD(struct_entry_name) \ - struct { \ - struct struct_entry_name *head; \ - struct struct_entry_name **tail; \ +#define HG_QUEUE_HEAD(struct_entry_name) \ + struct { \ + struct struct_entry_name *head; \ + struct struct_entry_name **tail; \ } -#define HG_QUEUE_ENTRY(struct_entry_name) \ - struct { \ - struct struct_entry_name *next; \ +#define HG_QUEUE_ENTRY(struct_entry_name) \ + struct { \ + struct struct_entry_name *next; \ } -#define HG_QUEUE_INIT(head_ptr) do { \ - (head_ptr)->head = NULL; \ - (head_ptr)->tail = &(head_ptr)->head; \ -} while (/*CONSTCOND*/0) +#define HG_QUEUE_INIT(head_ptr) \ + do { \ + (head_ptr)->head = NULL; \ + (head_ptr)->tail = &(head_ptr)->head; \ + } while (/*CONSTCOND*/ 0) -#define HG_QUEUE_IS_EMPTY(head_ptr) \ - ((head_ptr)->head == NULL) +#define HG_QUEUE_IS_EMPTY(head_ptr) ((head_ptr)->head == NULL) -#define HG_QUEUE_FIRST(head_ptr) \ - ((head_ptr)->head) +#define HG_QUEUE_FIRST(head_ptr) ((head_ptr)->head) -#define HG_QUEUE_NEXT(entry_ptr, entry_field_name) \ +#define HG_QUEUE_NEXT(entry_ptr, entry_field_name) \ ((entry_ptr)->entry_field_name.next) -#define HG_QUEUE_PUSH_TAIL(head_ptr, entry_ptr, entry_field_name) do { \ - (entry_ptr)->entry_field_name.next = NULL; \ - *(head_ptr)->tail = (entry_ptr); \ - (head_ptr)->tail = &(entry_ptr)->entry_field_name.next; \ -} while (/*CONSTCOND*/0) +#define HG_QUEUE_PUSH_TAIL(head_ptr, entry_ptr, entry_field_name) \ + do { \ + (entry_ptr)->entry_field_name.next = NULL; \ + *(head_ptr)->tail = (entry_ptr); \ + (head_ptr)->tail = &(entry_ptr)->entry_field_name.next; \ + } while (/*CONSTCOND*/ 0) /* TODO would be nice to not have any condition */ -#define HG_QUEUE_POP_HEAD(head_ptr, entry_field_name) do { \ - if ((head_ptr)->head && ((head_ptr)->head = (head_ptr)->head->entry_field_name.next) == NULL) \ - (head_ptr)->tail = &(head_ptr)->head; \ -} while (/*CONSTCOND*/0) - -#define HG_QUEUE_FOREACH(var, head_ptr, entry_field_name) \ - for ((var) = ((head_ptr)->head); \ - (var); \ - (var) = ((var)->entry_field_name.next)) +#define HG_QUEUE_POP_HEAD(head_ptr, entry_field_name) \ + do { \ + if ((head_ptr)->head && \ + ((head_ptr)->head = (head_ptr)->head->entry_field_name.next) == \ + NULL) \ + (head_ptr)->tail = &(head_ptr)->head; \ + } while (/*CONSTCOND*/ 0) + +#define HG_QUEUE_FOREACH(var, head_ptr, entry_field_name) \ + for ((var) = ((head_ptr)->head); (var); \ + (var) = ((var)->entry_field_name.next)) /** * Avoid using those for performance reasons or use mercury_list.h instead */ -#define HG_QUEUE_REMOVE(head_ptr, entry_ptr, type, entry_field_name) do { \ - if ((head_ptr)->head == (entry_ptr)) { \ - HG_QUEUE_POP_HEAD((head_ptr), entry_field_name); \ - } else { \ - struct type *curelm = (head_ptr)->head; \ - while (curelm->entry_field_name.next != (entry_ptr)) \ - curelm = curelm->entry_field_name.next; \ - if ((curelm->entry_field_name.next = \ - curelm->entry_field_name.next->entry_field_name.next) == NULL) \ - (head_ptr)->tail = &(curelm)->entry_field_name.next; \ - } \ -} while (/*CONSTCOND*/0) - +#define HG_QUEUE_REMOVE(head_ptr, entry_ptr, type, entry_field_name) \ + do { \ + if ((head_ptr)->head == (entry_ptr)) { \ + HG_QUEUE_POP_HEAD((head_ptr), entry_field_name); \ + } else { \ + struct type *curelm = (head_ptr)->head; \ + while (curelm->entry_field_name.next != (entry_ptr)) \ + curelm = curelm->entry_field_name.next; \ + if ((curelm->entry_field_name.next = \ + curelm->entry_field_name.next->entry_field_name \ + .next) == NULL) \ + (head_ptr)->tail = &(curelm)->entry_field_name.next; \ + } \ + } while (/*CONSTCOND*/ 0) #endif /* MERCURY_QUEUE_H */ diff --git a/src/util/mercury_request.c b/src/util/mercury_request.c index 12b1525f..ae4fb2af 100644 --- a/src/util/mercury_request.c +++ b/src/util/mercury_request.c @@ -9,13 +9,21 @@ */ #include "mercury_request.h" -#include "mercury_thread_mutex.h" #include "mercury_thread_condition.h" +#include "mercury_thread_mutex.h" #include "mercury_time.h" #include "mercury_util_error.h" #include +/****************/ +/* Local Macros */ +/****************/ + +/************************************/ +/* Local Type and Struct Definition */ +/************************************/ + struct hg_request_class { hg_request_progress_func_t progress_func; hg_request_trigger_func_t trigger_func; @@ -25,6 +33,14 @@ struct hg_request_class { hg_thread_cond_t progress_cond; }; +/********************/ +/* Local Prototypes */ +/********************/ + +/*******************/ +/* Local Variables */ +/*******************/ + /*---------------------------------------------------------------------------*/ static HG_UTIL_INLINE hg_util_bool_t hg_request_check(hg_request_t *request) @@ -34,13 +50,12 @@ hg_request_check(hg_request_t *request) hg_util_bool_t ret = HG_UTIL_FALSE; do { - trigger_ret = request->request_class->trigger_func(0, &trigger_flag, - request->request_class->arg); + trigger_ret = request->request_class->trigger_func( + 0, &trigger_flag, request->request_class->arg); } while ((trigger_ret == HG_UTIL_SUCCESS) && trigger_flag); - if (hg_atomic_cas32(&request->completed, HG_UTIL_TRUE, HG_UTIL_FALSE)) { + if (hg_atomic_cas32(&request->completed, HG_UTIL_TRUE, HG_UTIL_FALSE)) ret = HG_UTIL_TRUE; - } return ret; } @@ -48,16 +63,14 @@ hg_request_check(hg_request_t *request) /*---------------------------------------------------------------------------*/ hg_request_class_t * hg_request_init(hg_request_progress_func_t progress_func, - hg_request_trigger_func_t trigger_func, void *arg) + hg_request_trigger_func_t trigger_func, void *arg) { struct hg_request_class *hg_request_class = NULL; - hg_request_class = (struct hg_request_class *) - malloc(sizeof(struct hg_request_class)); - if (!hg_request_class) { - HG_UTIL_LOG_ERROR("Could not allocate hg_request_class"); - goto done; - } + hg_request_class = + (struct hg_request_class *) malloc(sizeof(struct hg_request_class)); + HG_UTIL_CHECK_ERROR_NORET( + hg_request_class == NULL, done, "Could not allocate hg_request_class"); hg_request_class->progress_func = progress_func; hg_request_class->trigger_func = trigger_func; @@ -74,17 +87,17 @@ hg_request_init(hg_request_progress_func_t progress_func, int hg_request_finalize(hg_request_class_t *request_class, void **arg) { - int ret = HG_UTIL_SUCCESS; - - if (!request_class) goto done; + if (!request_class) + goto done; - if (arg) *arg = request_class->arg; + if (arg) + *arg = request_class->arg; hg_thread_mutex_destroy(&request_class->progress_mutex); hg_thread_cond_destroy(&request_class->progress_cond); free(request_class); done: - return ret; + return HG_UTIL_SUCCESS; } /*---------------------------------------------------------------------------*/ @@ -94,10 +107,8 @@ hg_request_create(hg_request_class_t *request_class) struct hg_request *hg_request = NULL; hg_request = (struct hg_request *) malloc(sizeof(struct hg_request)); - if (!hg_request) { - HG_UTIL_LOG_ERROR("Could not allocate hg_request"); - goto done; - } + HG_UTIL_CHECK_ERROR_NORET( + hg_request == NULL, done, "Could not allocate hg_request"); hg_request->data = NULL; hg_atomic_set32(&hg_request->completed, HG_UTIL_FALSE); @@ -146,7 +157,8 @@ hg_request_destroy(hg_request_t *request) int hg_request_wait(hg_request_t *request, unsigned int timeout, unsigned int *flag) { - double remaining = timeout / 1000.0; /* Convert timeout in ms into seconds */ + double remaining = + timeout / 1000.0; /* Convert timeout in ms into seconds */ hg_util_bool_t completed = HG_UTIL_FALSE; int ret = HG_UTIL_SUCCESS; @@ -156,7 +168,8 @@ hg_request_wait(hg_request_t *request, unsigned int timeout, unsigned int *flag) hg_time_t t3, t4; completed = hg_request_check(request); - if (completed) break; + if (completed) + break; if (request->request_class->progressing) { hg_time_t t1, t2; @@ -189,8 +202,7 @@ hg_request_wait(hg_request_t *request, unsigned int timeout, unsigned int *flag) hg_time_get_current(&t3); request->request_class->progress_func( - (unsigned int) (remaining * 1000.0), - request->request_class->arg); + (unsigned int) (remaining * 1000.0), request->request_class->arg); if (timeout) { hg_time_get_current(&t4); @@ -205,7 +217,8 @@ hg_request_wait(hg_request_t *request, unsigned int timeout, unsigned int *flag) hg_thread_mutex_unlock(&request->request_class->progress_mutex); - if (flag) *flag = completed; + if (flag) + *flag = completed; return ret; } diff --git a/src/util/mercury_request.h b/src/util/mercury_request.h index a3ccbddc..e84bea3d 100644 --- a/src/util/mercury_request.h +++ b/src/util/mercury_request.h @@ -12,6 +12,7 @@ #define MERCURY_REQUEST_H #include "mercury_util_config.h" + #include "mercury_atomic.h" /** @@ -22,8 +23,8 @@ * (or HG_Bulk_trigger). */ -typedef struct hg_request_class hg_request_class_t; /* Opaque request class */ -typedef struct hg_request hg_request_t; /* Opaque request object */ +typedef struct hg_request_class hg_request_class_t; /* Opaque request class */ +typedef struct hg_request hg_request_t; /* Opaque request object */ struct hg_request { void *data; @@ -52,8 +53,8 @@ typedef int (*hg_request_progress_func_t)(unsigned int timeout, void *arg); * * \return HG_UTIL_SUCCESS or corresponding error code */ -typedef int (*hg_request_trigger_func_t)(unsigned int timeout, - unsigned int *flag, void *arg); +typedef int (*hg_request_trigger_func_t)( + unsigned int timeout, unsigned int *flag, void *arg); #ifdef __cplusplus extern "C" { @@ -70,9 +71,9 @@ extern "C" { * * \return Pointer to request class or NULL in case of failure */ -HG_UTIL_EXPORT hg_request_class_t * +HG_UTIL_PUBLIC hg_request_class_t * hg_request_init(hg_request_progress_func_t progress, - hg_request_trigger_func_t trigger, void *arg); + hg_request_trigger_func_t trigger, void *arg); /** * Finalize the request class. User args that were passed through @@ -81,7 +82,7 @@ hg_request_init(hg_request_progress_func_t progress, * \param request_class [IN] pointer to request class * \param arg [IN/OUT] pointer to init args */ -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_request_finalize(hg_request_class_t *request_class, void **arg); /** @@ -94,7 +95,7 @@ hg_request_finalize(hg_request_class_t *request_class, void **arg); * * \return Pointer to request or NULL in case of failure */ -HG_UTIL_EXPORT hg_request_t * +HG_UTIL_PUBLIC hg_request_t * hg_request_create(hg_request_class_t *request_class); /** @@ -104,7 +105,7 @@ hg_request_create(hg_request_class_t *request_class); * * \return Non-negative on success or negative on failure */ -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_request_destroy(hg_request_t *request); /** @@ -137,9 +138,9 @@ hg_request_complete(hg_request_t *request); * * \return Non-negative on success or negative on failure */ -HG_UTIL_EXPORT int -hg_request_wait(hg_request_t *request, unsigned int timeout, - unsigned int *flag); +HG_UTIL_PUBLIC int +hg_request_wait( + hg_request_t *request, unsigned int timeout, unsigned int *flag); /** * Wait timeout ms for all the specified request to complete. @@ -152,19 +153,8 @@ hg_request_wait(hg_request_t *request, unsigned int timeout, * \return Non-negative on success or negative on failure */ static HG_UTIL_INLINE int -hg_request_waitall(int count, hg_request_t *request[], unsigned int timeout, - unsigned int *flag); - -/** - * A convenience callback function to pass to HG calls when using - * hg_request. This function simply interprets the argument passed into - * the HG call as an hg_request_t and calls hg_request_complete on it. - * - * \return HG_SUCCESS if hg_request_complete returns HG_UTIL_SUCCESS, - * HG_OTHER_ERROR otherwise - */ -//HG_UTIL_EXPORT hg_return_t -//hg_request_complete_cb(const struct hg_cb_info *cb_info); +hg_request_waitall(int count, hg_request_t *request[], unsigned int timeout, + unsigned int *flag); /** * Attach user data to a specified request. @@ -194,7 +184,7 @@ hg_request_get_data(hg_request_t *request); * * \return Non-negative on success or negative on failure * -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_request_cancel(hg_request_t *request); */ @@ -202,64 +192,47 @@ hg_request_cancel(hg_request_t *request); static HG_UTIL_INLINE int hg_request_reset(hg_request_t *request) { - int ret = HG_UTIL_SUCCESS; - hg_atomic_set32(&request->completed, HG_UTIL_FALSE); - return ret; + return HG_UTIL_SUCCESS; } /*---------------------------------------------------------------------------*/ static HG_UTIL_INLINE int hg_request_complete(hg_request_t *request) { - int ret = HG_UTIL_SUCCESS; - hg_atomic_incr32(&request->completed); - return ret; + return HG_UTIL_SUCCESS; } /*---------------------------------------------------------------------------*/ static HG_UTIL_INLINE int hg_request_waitall(int count, hg_request_t *request[], unsigned int timeout, - unsigned int *flag) + unsigned int *flag) { - /* TODO */ int i; + for (i = 0; i < count; i++) hg_request_wait(request[i], timeout, flag); + return HG_UTIL_SUCCESS; } -/*---------------------------------------------------------------------------*/ -//hg_return_t -//hg_request_complete_cb(const struct hg_cb_info *cb_info) -//{ -// int ret = hg_request_complete(cb_info->arg); -// return ret == HG_UTIL_SUCCESS ? HG_SUCCESS : HG_OTHER_ERROR; -//} - /*---------------------------------------------------------------------------*/ static HG_UTIL_INLINE int hg_request_set_data(hg_request_t *request, void *data) { - int ret = HG_UTIL_SUCCESS; - request->data = data; - return ret; + return HG_UTIL_SUCCESS; } /*---------------------------------------------------------------------------*/ static HG_UTIL_INLINE void * hg_request_get_data(hg_request_t *request) { - void *ret = NULL; - - ret = request->data; - - return ret; + return request->data; } #ifdef __cplusplus diff --git a/src/util/mercury_thread.c b/src/util/mercury_thread.c index fcba0e54..1c0e9769 100644 --- a/src/util/mercury_thread.c +++ b/src/util/mercury_thread.c @@ -25,19 +25,16 @@ hg_thread_init(hg_thread_t *thread) int hg_thread_create(hg_thread_t *thread, hg_thread_func_t f, void *data) { - int ret = HG_UTIL_SUCCESS; - #ifdef _WIN32 *thread = CreateThread(NULL, 0, f, data, 0, NULL); - if (*thread == NULL) ret = HG_UTIL_FAIL; + if (*thread == NULL) + return HG_UTIL_FAIL; #else - if (pthread_create(thread, NULL, f, data)) { - HG_UTIL_LOG_ERROR("pthread_create() failed"); - ret = HG_UTIL_FAIL; - } + if (pthread_create(thread, NULL, f, data)) + return HG_UTIL_FAIL; #endif - return ret; + return HG_UTIL_SUCCESS; } /*---------------------------------------------------------------------------*/ @@ -55,46 +52,36 @@ hg_thread_exit(hg_thread_ret_t ret) int hg_thread_join(hg_thread_t thread) { - int ret = HG_UTIL_SUCCESS; - #ifdef _WIN32 WaitForSingleObject(thread, INFINITE); CloseHandle(thread); #else - if (pthread_join(thread, NULL)) { - HG_UTIL_LOG_ERROR("pthread_join() failed"); - ret = HG_UTIL_FAIL; - } + if (pthread_join(thread, NULL)) + return HG_UTIL_FAIL; #endif - return ret; + return HG_UTIL_SUCCESS; } /*---------------------------------------------------------------------------*/ int hg_thread_cancel(hg_thread_t thread) { - int ret = HG_UTIL_SUCCESS; - #ifdef _WIN32 WaitForSingleObject(thread, 0); CloseHandle(thread); #else - if (pthread_cancel(thread)) { - HG_UTIL_LOG_ERROR("pthread_cancel() failed"); - ret = HG_UTIL_FAIL; - } + if (pthread_cancel(thread)) + return HG_UTIL_FAIL; #endif - return ret; + return HG_UTIL_SUCCESS; } /*---------------------------------------------------------------------------*/ int hg_thread_yield(void) { - int ret = HG_UTIL_SUCCESS; - #ifdef _WIN32 SwitchToThread(); #elif defined(__APPLE__) @@ -103,98 +90,73 @@ hg_thread_yield(void) pthread_yield(); #endif - return ret; + return HG_UTIL_SUCCESS; } /*---------------------------------------------------------------------------*/ int hg_thread_key_create(hg_thread_key_t *key) { - int ret = HG_UTIL_SUCCESS; - - if (!key) { - HG_UTIL_LOG_ERROR("NULL pointer to hg_thread_key_t"); - ret = HG_UTIL_FAIL; - return ret; - } + if (!key) + return HG_UTIL_FAIL; #ifdef _WIN32 - if ((*key = TlsAlloc()) == TLS_OUT_OF_INDEXES) { - HG_UTIL_LOG_ERROR("TlsAlloc() failed"); - ret = HG_UTIL_FAIL; - } + if ((*key = TlsAlloc()) == TLS_OUT_OF_INDEXES) + return HG_UTIL_FAIL; #else - if (pthread_key_create(key, NULL)) { - HG_UTIL_LOG_ERROR("pthread_key_create() failed"); - ret = HG_UTIL_FAIL; - } + if (pthread_key_create(key, NULL)) + return HG_UTIL_FAIL; #endif - return ret; + return HG_UTIL_SUCCESS; } /*---------------------------------------------------------------------------*/ int hg_thread_key_delete(hg_thread_key_t key) { - int ret = HG_UTIL_SUCCESS; - #ifdef _WIN32 - if (!TlsFree(key)) { - HG_UTIL_LOG_ERROR("TlsFree() failed"); - ret = HG_UTIL_FAIL; - } + if (!TlsFree(key)) + return HG_UTIL_FAIL; #else - if (pthread_key_delete(key)) { - HG_UTIL_LOG_ERROR("pthread_key_delete() failed"); - ret = HG_UTIL_FAIL; - } + if (pthread_key_delete(key)) + return HG_UTIL_FAIL; #endif - return ret; + return HG_UTIL_SUCCESS; } /*---------------------------------------------------------------------------*/ int hg_thread_getaffinity(hg_thread_t thread, hg_cpu_set_t *cpu_mask) { - int ret = HG_UTIL_SUCCESS; - #if defined(_WIN32) - HG_UTIL_LOG_ERROR("not supported"); + return HG_UTIL_FAIL; #elif defined(__APPLE__) - (void)thread; - (void)cpu_mask; + (void) thread; + (void) cpu_mask; + return HG_UTIL_FAIL; #else - if (pthread_getaffinity_np(thread, sizeof(hg_cpu_set_t), cpu_mask)) { - HG_UTIL_LOG_ERROR("pthread_getaffinity_np() failed"); - ret = HG_UTIL_FAIL; - } + if (pthread_getaffinity_np(thread, sizeof(hg_cpu_set_t), cpu_mask)) + return HG_UTIL_FAIL; + return HG_UTIL_SUCCESS; #endif - - return ret; } /*---------------------------------------------------------------------------*/ int hg_thread_setaffinity(hg_thread_t thread, const hg_cpu_set_t *cpu_mask) { - int ret = HG_UTIL_SUCCESS; - #if defined(_WIN32) - if (!SetThreadAffinityMask(thread, *cpu_mask)) { - HG_UTIL_LOG_ERROR("SetThreadAffinityMask() failed"); - ret = HG_UTIL_FAIL; - } + if (!SetThreadAffinityMask(thread, *cpu_mask)) + return HG_UTIL_FAIL; #elif defined(__APPLE__) - (void)thread; - (void)cpu_mask; + (void) thread; + (void) cpu_mask; + return HG_UTIL_FAIL; #else - if (pthread_setaffinity_np(thread, sizeof(hg_cpu_set_t), cpu_mask)) { - HG_UTIL_LOG_ERROR("pthread_setaffinity_np() failed"); - ret = HG_UTIL_FAIL; - } + if (pthread_setaffinity_np(thread, sizeof(hg_cpu_set_t), cpu_mask)) + return HG_UTIL_FAIL; + return HG_UTIL_SUCCESS; #endif - - return ret; } diff --git a/src/util/mercury_thread.h b/src/util/mercury_thread.h index a90855c8..cc4bbf14 100644 --- a/src/util/mercury_thread.h +++ b/src/util/mercury_thread.h @@ -12,38 +12,37 @@ #define MERCURY_THREAD_H #if !defined(_WIN32) && !defined(_GNU_SOURCE) -# define _GNU_SOURCE +# define _GNU_SOURCE #endif #include "mercury_util_config.h" -#include "mercury_util_error.h" #ifdef _WIN32 -# include +# include typedef HANDLE hg_thread_t; typedef LPTHREAD_START_ROUTINE hg_thread_func_t; typedef DWORD hg_thread_ret_t; -# define HG_THREAD_RETURN_TYPE hg_thread_ret_t WINAPI +# define HG_THREAD_RETURN_TYPE hg_thread_ret_t WINAPI typedef DWORD hg_thread_key_t; typedef DWORD_PTR hg_cpu_set_t; #else -# include +# include typedef pthread_t hg_thread_t; typedef void *(*hg_thread_func_t)(void *); typedef void *hg_thread_ret_t; -# define HG_THREAD_RETURN_TYPE hg_thread_ret_t +# define HG_THREAD_RETURN_TYPE hg_thread_ret_t typedef pthread_key_t hg_thread_key_t; -# ifdef __APPLE__ +# ifdef __APPLE__ /* Size definition for CPU sets. */ -# define HG_CPU_SETSIZE 1024 -# define HG_NCPUBITS (8 * sizeof (hg_cpu_mask_t)) +# define HG_CPU_SETSIZE 1024 +# define HG_NCPUBITS (8 * sizeof(hg_cpu_mask_t)) /* Type for array elements in 'cpu_set_t'. */ typedef hg_util_uint64_t hg_cpu_mask_t; typedef struct { hg_cpu_mask_t bits[HG_CPU_SETSIZE / HG_NCPUBITS]; } hg_cpu_set_t; -# else +# else typedef cpu_set_t hg_cpu_set_t; -# endif +# endif #endif #ifdef __cplusplus @@ -55,7 +54,7 @@ extern "C" { * * \param thread [IN/OUT] pointer to thread object */ -HG_UTIL_EXPORT void +HG_UTIL_PUBLIC void hg_thread_init(hg_thread_t *thread); /** @@ -67,7 +66,7 @@ hg_thread_init(hg_thread_t *thread); * * \return Non-negative on success or negative on failure */ -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_thread_create(hg_thread_t *thread, hg_thread_func_t f, void *data); /** @@ -77,7 +76,7 @@ hg_thread_create(hg_thread_t *thread, hg_thread_func_t f, void *data); * * \return Non-negative on success or negative on failure */ -HG_UTIL_EXPORT void +HG_UTIL_PUBLIC void hg_thread_exit(hg_thread_ret_t ret); /** @@ -87,7 +86,7 @@ hg_thread_exit(hg_thread_ret_t ret); * * \return Non-negative on success or negative on failure */ -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_thread_join(hg_thread_t thread); /** @@ -97,7 +96,7 @@ hg_thread_join(hg_thread_t thread); * * \return Non-negative on success or negative on failure */ -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_thread_cancel(hg_thread_t thread); /** @@ -105,7 +104,7 @@ hg_thread_cancel(hg_thread_t thread); * * \return Non-negative on success or negative on failure */ -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_thread_yield(void); /** @@ -131,7 +130,7 @@ hg_thread_equal(hg_thread_t t1, hg_thread_t t2); * * \return Non-negative on success or negative on failure */ -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_thread_key_create(hg_thread_key_t *key); /** @@ -142,7 +141,7 @@ hg_thread_key_create(hg_thread_key_t *key); * * \return Non-negative on success or negative on failure */ -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_thread_key_delete(hg_thread_key_t key); /** @@ -174,7 +173,7 @@ hg_thread_setspecific(hg_thread_key_t key, const void *value); * * \return Non-negative on success or negative on failure */ -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_thread_getaffinity(hg_thread_t thread, hg_cpu_set_t *cpu_mask); /** @@ -185,7 +184,7 @@ hg_thread_getaffinity(hg_thread_t thread, hg_cpu_set_t *cpu_mask); * * \return Non-negative on success or negative on failure */ -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_thread_setaffinity(hg_thread_t thread, const hg_cpu_set_t *cpu_mask); /*---------------------------------------------------------------------------*/ @@ -214,36 +213,26 @@ hg_thread_equal(hg_thread_t t1, hg_thread_t t2) static HG_UTIL_INLINE void * hg_thread_getspecific(hg_thread_key_t key) { - void *ret; - #ifdef _WIN32 - ret = TlsGetValue(key); + return TlsGetValue(key); #else - ret = pthread_getspecific(key); + return pthread_getspecific(key); #endif - - return ret; } /*---------------------------------------------------------------------------*/ static HG_UTIL_INLINE int hg_thread_setspecific(hg_thread_key_t key, const void *value) { - int ret = HG_UTIL_SUCCESS; - #ifdef _WIN32 - if (!TlsSetValue(key, (LPVOID) value)) { - HG_UTIL_LOG_ERROR("TlsSetValue() failed"); - ret = HG_UTIL_FAIL; - } + if (!TlsSetValue(key, (LPVOID) value)) + return HG_UTIL_FAIL; #else - if (pthread_setspecific(key, value)) { - HG_UTIL_LOG_ERROR("pthread_setspecific() failed"); - ret = HG_UTIL_FAIL; - } + if (pthread_setspecific(key, value)) + return HG_UTIL_FAIL; #endif - return ret; + return HG_UTIL_SUCCESS; } #ifdef __cplusplus diff --git a/src/util/mercury_thread_condition.c b/src/util/mercury_thread_condition.c index 7771f85f..76e4fef5 100644 --- a/src/util/mercury_thread_condition.c +++ b/src/util/mercury_thread_condition.c @@ -14,34 +14,33 @@ int hg_thread_cond_init(hg_thread_cond_t *cond) { - int ret = HG_UTIL_SUCCESS; - #ifdef _WIN32 InitializeConditionVariable(cond); #else pthread_condattr_t attr; pthread_condattr_init(&attr); -# if defined(HG_UTIL_HAS_PTHREAD_CONDATTR_SETCLOCK) && defined(HG_UTIL_HAS_CLOCK_MONOTONIC) +# if defined(HG_UTIL_HAS_PTHREAD_CONDATTR_SETCLOCK) && \ + defined(HG_UTIL_HAS_CLOCK_MONOTONIC) /* Must set clock ID if using different clock */ pthread_condattr_setclock(&attr, CLOCK_MONOTONIC); -# endif - if (pthread_cond_init(cond, &attr)) ret = HG_UTIL_FAIL; +# endif + if (pthread_cond_init(cond, &attr)) + return HG_UTIL_FAIL; pthread_condattr_destroy(&attr); #endif - return ret; + return HG_UTIL_SUCCESS; } /*---------------------------------------------------------------------------*/ int hg_thread_cond_destroy(hg_thread_cond_t *cond) { - int ret = HG_UTIL_SUCCESS; - #ifndef _WIN32 - if (pthread_cond_destroy(cond)) ret = HG_UTIL_FAIL; + if (pthread_cond_destroy(cond)) + return HG_UTIL_FAIL; #endif - return ret; + return HG_UTIL_SUCCESS; } diff --git a/src/util/mercury_thread_condition.h b/src/util/mercury_thread_condition.h index 8c6c04de..70e9748e 100644 --- a/src/util/mercury_thread_condition.h +++ b/src/util/mercury_thread_condition.h @@ -16,12 +16,12 @@ #ifdef _WIN32 typedef CONDITION_VARIABLE hg_thread_cond_t; #else -# if defined(HG_UTIL_HAS_PTHREAD_CONDATTR_SETCLOCK) -# include "mercury_time.h" -# elif defined(HG_UTIL_HAS_SYSTIME_H) -# include -# endif -# include +# if defined(HG_UTIL_HAS_PTHREAD_CONDATTR_SETCLOCK) +# include "mercury_time.h" +# elif defined(HG_UTIL_HAS_SYSTIME_H) +# include +# endif +# include typedef pthread_cond_t hg_thread_cond_t; #endif @@ -36,7 +36,7 @@ extern "C" { * * \return Non-negative on success or negative on failure */ -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_thread_cond_init(hg_thread_cond_t *cond); /** @@ -46,7 +46,7 @@ hg_thread_cond_init(hg_thread_cond_t *cond); * * \return Non-negative on success or negative on failure */ -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_thread_cond_destroy(hg_thread_cond_t *cond); /** @@ -90,94 +90,89 @@ hg_thread_cond_wait(hg_thread_cond_t *cond, hg_thread_mutex_t *mutex); * \return Non-negative on success or negative on failure */ static HG_UTIL_INLINE int -hg_thread_cond_timedwait(hg_thread_cond_t *cond, hg_thread_mutex_t *mutex, - unsigned int timeout); +hg_thread_cond_timedwait( + hg_thread_cond_t *cond, hg_thread_mutex_t *mutex, unsigned int timeout); /*---------------------------------------------------------------------------*/ static HG_UTIL_INLINE int hg_thread_cond_signal(hg_thread_cond_t *cond) { - int ret = HG_UTIL_SUCCESS; - #ifdef _WIN32 WakeConditionVariable(cond); #else - if (pthread_cond_signal(cond)) ret = HG_UTIL_FAIL; + if (pthread_cond_signal(cond)) + return HG_UTIL_FAIL; #endif - return ret; + return HG_UTIL_SUCCESS; } /*---------------------------------------------------------------------------*/ static HG_UTIL_INLINE int hg_thread_cond_broadcast(hg_thread_cond_t *cond) { - int ret = HG_UTIL_SUCCESS; - #ifdef _WIN32 WakeAllConditionVariable(cond); #else - if (pthread_cond_broadcast(cond)) ret = HG_UTIL_FAIL; + if (pthread_cond_broadcast(cond)) + return HG_UTIL_FAIL; #endif - return ret; + return HG_UTIL_SUCCESS; } /*---------------------------------------------------------------------------*/ static HG_UTIL_INLINE int hg_thread_cond_wait(hg_thread_cond_t *cond, hg_thread_mutex_t *mutex) { - int ret = HG_UTIL_SUCCESS; - #ifdef _WIN32 - if (!SleepConditionVariableCS(cond, mutex, INFINITE)) ret = HG_UTIL_FAIL; + if (!SleepConditionVariableCS(cond, mutex, INFINITE)) + return HG_UTIL_FAIL; #else - if (pthread_cond_wait(cond, mutex)) ret = HG_UTIL_FAIL; + if (pthread_cond_wait(cond, mutex)) + return HG_UTIL_FAIL; #endif - return ret; + return HG_UTIL_SUCCESS; } /*---------------------------------------------------------------------------*/ static HG_UTIL_INLINE int -hg_thread_cond_timedwait(hg_thread_cond_t *cond, hg_thread_mutex_t *mutex, - unsigned int timeout) +hg_thread_cond_timedwait( + hg_thread_cond_t *cond, hg_thread_mutex_t *mutex, unsigned int timeout) { - int ret = HG_UTIL_SUCCESS; - #ifdef _WIN32 - if (!SleepConditionVariableCS(cond, mutex, timeout)) ret = HG_UTIL_FAIL; + if (!SleepConditionVariableCS(cond, mutex, timeout)) + return HG_UTIL_FAIL; #else - int pret; -# if defined(HG_UTIL_HAS_PTHREAD_CONDATTR_SETCLOCK) +# if defined(HG_UTIL_HAS_PTHREAD_CONDATTR_SETCLOCK) hg_time_t now; -# elif defined(HG_UTIL_HAS_SYSTIME_H) +# elif defined(HG_UTIL_HAS_SYSTIME_H) struct timeval now; -# endif +# endif struct timespec abs_timeout; long int abs_timeout_us; ldiv_t ld; /* Need to convert timeout (ms) to absolute time */ -# if defined(HG_UTIL_HAS_PTHREAD_CONDATTR_SETCLOCK) +# if defined(HG_UTIL_HAS_PTHREAD_CONDATTR_SETCLOCK) if (hg_time_get_current(&now) != HG_UTIL_SUCCESS) return HG_UTIL_FAIL; -# elif defined(HG_UTIL_HAS_SYSTIME_H) +# elif defined(HG_UTIL_HAS_SYSTIME_H) if (gettimeofday(&now, NULL) != 0) return HG_UTIL_FAIL; -# endif +# endif abs_timeout_us = now.tv_usec + timeout * 1000L; /* Get sec / nsec */ ld = ldiv(abs_timeout_us, 1000000L); abs_timeout.tv_sec = now.tv_sec + ld.quot; abs_timeout.tv_nsec = ld.rem * 1000L; - pret = pthread_cond_timedwait(cond, mutex, &abs_timeout); - if (pret) - ret = HG_UTIL_FAIL; + if (pthread_cond_timedwait(cond, mutex, &abs_timeout)) + return HG_UTIL_FAIL; #endif - return ret; + return HG_UTIL_SUCCESS; } #ifdef __cplusplus diff --git a/src/util/mercury_thread_mutex.c b/src/util/mercury_thread_mutex.c index 44aa0c1b..d5fcc7ce 100644 --- a/src/util/mercury_thread_mutex.c +++ b/src/util/mercury_thread_mutex.c @@ -14,39 +14,37 @@ int hg_thread_mutex_init(hg_thread_mutex_t *mutex) { - int ret = HG_UTIL_SUCCESS; - #ifdef _WIN32 InitializeCriticalSection(mutex); #else pthread_mutexattr_t mutex_attr; pthread_mutexattr_init(&mutex_attr); -#ifdef HG_UTIL_HAS_PTHREAD_MUTEX_ADAPTIVE_NP +# ifdef HG_UTIL_HAS_PTHREAD_MUTEX_ADAPTIVE_NP /* Set type to PTHREAD_MUTEX_ADAPTIVE_NP to improve performance */ pthread_mutexattr_settype(&mutex_attr, PTHREAD_MUTEX_ADAPTIVE_NP); -#else +# else pthread_mutexattr_settype(&mutex_attr, PTHREAD_MUTEX_DEFAULT); -#endif - if (pthread_mutex_init(mutex, &mutex_attr)) ret = HG_UTIL_FAIL; +# endif + if (pthread_mutex_init(mutex, &mutex_attr)) + return HG_UTIL_FAIL; pthread_mutexattr_destroy(&mutex_attr); #endif - return ret; + return HG_UTIL_SUCCESS; } /*---------------------------------------------------------------------------*/ int hg_thread_mutex_destroy(hg_thread_mutex_t *mutex) { - int ret = HG_UTIL_SUCCESS; - #ifdef _WIN32 DeleteCriticalSection(mutex); #else - if (pthread_mutex_destroy(mutex)) ret = HG_UTIL_FAIL; + if (pthread_mutex_destroy(mutex)) + return HG_UTIL_FAIL; #endif - return ret; + return HG_UTIL_SUCCESS; } diff --git a/src/util/mercury_thread_mutex.h b/src/util/mercury_thread_mutex.h index b846b106..fe54a0cf 100644 --- a/src/util/mercury_thread_mutex.h +++ b/src/util/mercury_thread_mutex.h @@ -12,14 +12,14 @@ #define MERCURY_THREAD_MUTEX_H #include "mercury_util_config.h" + #ifdef _WIN32 -# include -# define HG_THREAD_MUTEX_INITIALIZER NULL +# include +# define HG_THREAD_MUTEX_INITIALIZER NULL typedef CRITICAL_SECTION hg_thread_mutex_t; #else -# include -# include -# define HG_THREAD_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER +# include +# define HG_THREAD_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER typedef pthread_mutex_t hg_thread_mutex_t; #endif @@ -34,7 +34,7 @@ extern "C" { * * \return Non-negative on success or negative on failure */ -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_thread_mutex_init(hg_thread_mutex_t *mutex); /** @@ -44,7 +44,7 @@ hg_thread_mutex_init(hg_thread_mutex_t *mutex); * * \return Non-negative on success or negative on failure */ -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_thread_mutex_destroy(hg_thread_mutex_t *mutex); /** @@ -81,45 +81,43 @@ hg_thread_mutex_unlock(hg_thread_mutex_t *mutex); static HG_UTIL_INLINE int hg_thread_mutex_lock(hg_thread_mutex_t *mutex) { - int ret = HG_UTIL_SUCCESS; - #ifdef _WIN32 EnterCriticalSection(mutex); #else - if (pthread_mutex_lock(mutex)) ret = HG_UTIL_FAIL; + if (pthread_mutex_lock(mutex)) + return HG_UTIL_FAIL; #endif - return ret; + return HG_UTIL_SUCCESS; } /*---------------------------------------------------------------------------*/ static HG_UTIL_INLINE int hg_thread_mutex_try_lock(hg_thread_mutex_t *mutex) { - int ret = HG_UTIL_SUCCESS; - #ifdef _WIN32 - if (!TryEnterCriticalSection(mutex)) ret = HG_UTIL_FAIL; + if (!TryEnterCriticalSection(mutex)) + return HG_UTIL_FAIL; #else - if (pthread_mutex_trylock(mutex)) ret = HG_UTIL_FAIL; + if (pthread_mutex_trylock(mutex)) + return HG_UTIL_FAIL; #endif - return ret; + return HG_UTIL_SUCCESS; } /*---------------------------------------------------------------------------*/ static HG_UTIL_INLINE int hg_thread_mutex_unlock(hg_thread_mutex_t *mutex) { - int ret = HG_UTIL_SUCCESS; - #ifdef _WIN32 LeaveCriticalSection(mutex); #else - if (pthread_mutex_unlock(mutex)) ret = HG_UTIL_FAIL; + if (pthread_mutex_unlock(mutex)) + return HG_UTIL_FAIL; #endif - return ret; + return HG_UTIL_SUCCESS; } #ifdef __cplusplus diff --git a/src/util/mercury_thread_pool.c b/src/util/mercury_thread_pool.c index d68d00f3..48e3b6ce 100644 --- a/src/util/mercury_thread_pool.c +++ b/src/util/mercury_thread_pool.c @@ -10,22 +10,44 @@ #include "mercury_thread_pool.h" +#include "mercury_util_error.h" + #include +/****************/ +/* Local Macros */ +/****************/ + +/************************************/ +/* Local Type and Struct Definition */ +/************************************/ + struct hg_thread_pool_private { struct hg_thread_pool pool; unsigned int thread_count; hg_thread_t *threads; }; +/********************/ +/* Local Prototypes */ +/********************/ + /** * Worker thread run by the thread pool */ static HG_THREAD_RETURN_TYPE +hg_thread_pool_worker(void *args); + +/*******************/ +/* Local Variables */ +/*******************/ + +/*---------------------------------------------------------------------------*/ +static HG_THREAD_RETURN_TYPE hg_thread_pool_worker(void *args) { hg_thread_ret_t ret = 0; - hg_thread_pool_t *pool = (hg_thread_pool_t*) args; + hg_thread_pool_t *pool = (hg_thread_pool_t *) args; struct hg_thread_work *work; while (1) { @@ -33,17 +55,19 @@ hg_thread_pool_worker(void *args) /* If not shutting down and nothing to do, worker sleeps */ while (!pool->shutdown && HG_QUEUE_IS_EMPTY(&pool->queue)) { + int rc; + pool->sleeping_worker_count++; - if (hg_thread_cond_wait(&pool->cond, &pool->mutex) != HG_UTIL_SUCCESS) { - HG_UTIL_LOG_ERROR("Thread cannot wait on condition variable"); - goto unlock; - } + + rc = hg_thread_cond_wait(&pool->cond, &pool->mutex); + HG_UTIL_CHECK_ERROR_NORET(rc != HG_UTIL_SUCCESS, unlock, + "Thread cannot wait on condition variable"); + pool->sleeping_worker_count--; } - if (pool->shutdown && HG_QUEUE_IS_EMPTY(&pool->queue)) { + if (pool->shutdown && HG_QUEUE_IS_EMPTY(&pool->queue)) goto unlock; - } /* Grab our task */ work = HG_QUEUE_FIRST(&pool->queue); @@ -58,7 +82,7 @@ hg_thread_pool_worker(void *args) unlock: hg_thread_mutex_unlock(&pool->mutex); - hg_thread_exit(ret); + return ret; } @@ -66,66 +90,53 @@ hg_thread_pool_worker(void *args) int hg_thread_pool_init(unsigned int thread_count, hg_thread_pool_t **pool_ptr) { - int ret = HG_UTIL_SUCCESS; + int ret = HG_UTIL_SUCCESS, rc; struct hg_thread_pool_private *priv_pool = NULL; unsigned int i; - if (!pool_ptr) { - HG_UTIL_LOG_ERROR("Cannot pass NULL pointer"); - ret = HG_UTIL_FAIL; - goto done; - } + HG_UTIL_CHECK_ERROR( + pool_ptr == NULL, error, ret, HG_UTIL_FAIL, "NULL pointer"); priv_pool = (struct hg_thread_pool_private *) malloc( sizeof(struct hg_thread_pool_private)); - if (!priv_pool) { - HG_UTIL_LOG_ERROR("Could not allocate thread pool"); - ret = HG_UTIL_FAIL; - goto done; - } + HG_UTIL_CHECK_ERROR(priv_pool == NULL, error, ret, HG_UTIL_FAIL, + "Could not allocate thread pool"); + priv_pool->pool.sleeping_worker_count = 0; priv_pool->thread_count = thread_count; priv_pool->threads = NULL; HG_QUEUE_INIT(&priv_pool->pool.queue); priv_pool->pool.shutdown = 0; - if (hg_thread_mutex_init(&priv_pool->pool.mutex) != HG_UTIL_SUCCESS) { - HG_UTIL_LOG_ERROR("Could not initialize mutex"); - ret = HG_UTIL_FAIL; - goto done; - } - if (hg_thread_cond_init(&priv_pool->pool.cond) != HG_UTIL_SUCCESS) { - HG_UTIL_LOG_ERROR("Could not initialize thread condition"); - ret = HG_UTIL_FAIL; - goto done; - } + rc = hg_thread_mutex_init(&priv_pool->pool.mutex); + HG_UTIL_CHECK_ERROR(rc != HG_UTIL_SUCCESS, error, ret, HG_UTIL_FAIL, + "Could not initialize mutex"); - priv_pool->threads = (hg_thread_t*) malloc(thread_count * sizeof(hg_thread_t)); - if (!priv_pool->threads) { - HG_UTIL_LOG_ERROR("Could not allocate thread pool array"); - ret = HG_UTIL_FAIL; - goto done; - } + rc = hg_thread_cond_init(&priv_pool->pool.cond); + HG_UTIL_CHECK_ERROR(rc != HG_UTIL_SUCCESS, error, ret, HG_UTIL_FAIL, + "Could not initialize thread condition"); + + priv_pool->threads = + (hg_thread_t *) malloc(thread_count * sizeof(hg_thread_t)); + HG_UTIL_CHECK_ERROR(!priv_pool->threads, error, ret, HG_UTIL_FAIL, + "Could not allocate thread pool array"); /* Start worker threads */ for (i = 0; i < thread_count; i++) { - if (hg_thread_create(&priv_pool->threads[i], hg_thread_pool_worker, - (void*) priv_pool) != HG_UTIL_SUCCESS) { - HG_UTIL_LOG_ERROR("Could not create thread"); - ret = HG_UTIL_FAIL; - goto done; - } + rc = hg_thread_create( + &priv_pool->threads[i], hg_thread_pool_worker, (void *) priv_pool); + HG_UTIL_CHECK_ERROR(rc != HG_UTIL_SUCCESS, error, ret, HG_UTIL_FAIL, + "Could not create thread"); } *pool_ptr = (struct hg_thread_pool *) priv_pool; -done: - if (ret != HG_UTIL_SUCCESS) { - if (priv_pool) { - hg_thread_pool_destroy((struct hg_thread_pool *) priv_pool); - } - priv_pool = NULL; - } + return ret; + +error: + if (priv_pool) + hg_thread_pool_destroy((struct hg_thread_pool *) priv_pool); + return ret; } @@ -135,50 +146,46 @@ hg_thread_pool_destroy(hg_thread_pool_t *pool) { struct hg_thread_pool_private *priv_pool = (struct hg_thread_pool_private *) pool; - int ret = HG_UTIL_SUCCESS; + int ret = HG_UTIL_SUCCESS, rc; unsigned int i; - if (!priv_pool) goto done; + if (!priv_pool) + goto done; if (priv_pool->threads) { hg_thread_mutex_lock(&priv_pool->pool.mutex); priv_pool->pool.shutdown = 1; - if (hg_thread_cond_broadcast(&priv_pool->pool.cond) != HG_UTIL_SUCCESS) { - HG_UTIL_LOG_ERROR("Could not broadcast condition signal"); - ret = HG_UTIL_FAIL; - } + rc = hg_thread_cond_broadcast(&priv_pool->pool.cond); + HG_UTIL_CHECK_ERROR(rc != HG_UTIL_SUCCESS, error, ret, HG_UTIL_FAIL, + "Could not broadcast condition signal"); hg_thread_mutex_unlock(&priv_pool->pool.mutex); - if (ret != HG_UTIL_SUCCESS) goto done; - - for(i = 0; i < priv_pool->thread_count; i++) { - if (hg_thread_join(priv_pool->threads[i]) != HG_UTIL_SUCCESS) { - HG_UTIL_LOG_ERROR("Could not join thread"); - ret = HG_UTIL_FAIL; - goto done; - } + for (i = 0; i < priv_pool->thread_count; i++) { + rc = hg_thread_join(priv_pool->threads[i]); + HG_UTIL_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, HG_UTIL_FAIL, + "Could not join thread"); } } - free(priv_pool->threads); - priv_pool->threads = NULL; + rc = hg_thread_mutex_destroy(&priv_pool->pool.mutex); + HG_UTIL_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, HG_UTIL_FAIL, + "Could not destroy mutex"); - if (hg_thread_mutex_destroy(&priv_pool->pool.mutex) != HG_UTIL_SUCCESS) { - HG_UTIL_LOG_ERROR("Could not destroy mutex"); - ret = HG_UTIL_FAIL; - goto done; - } - if (hg_thread_cond_destroy(&priv_pool->pool.cond) != HG_UTIL_SUCCESS){ - HG_UTIL_LOG_ERROR("Could not destroy thread condition"); - ret = HG_UTIL_FAIL; - goto done; - } + rc = hg_thread_cond_destroy(&priv_pool->pool.cond); + HG_UTIL_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, HG_UTIL_FAIL, + "Could not destroy thread condition"); + free(priv_pool->threads); free(priv_pool); done: return ret; + +error: + hg_thread_mutex_unlock(&priv_pool->pool.mutex); + + return ret; } diff --git a/src/util/mercury_thread_pool.h b/src/util/mercury_thread_pool.h index d9914ae4..9f8433a9 100644 --- a/src/util/mercury_thread_pool.h +++ b/src/util/mercury_thread_pool.h @@ -11,10 +11,13 @@ #ifndef MERCURY_THREAD_POOL_H #define MERCURY_THREAD_POOL_H -#include "mercury_thread.h" #include "mercury_queue.h" +#include "mercury_thread.h" #include "mercury_thread_condition.h" -#include "mercury_util_error.h" + +/*************************************/ +/* Public Type and Struct Definition */ +/*************************************/ typedef struct hg_thread_pool hg_thread_pool_t; @@ -32,6 +35,14 @@ struct hg_thread_work { HG_QUEUE_ENTRY(hg_thread_work) entry; /* Internal */ }; +/*****************/ +/* Public Macros */ +/*****************/ + +/*********************/ +/* Public Prototypes */ +/*********************/ + #ifdef __cplusplus extern "C" { #endif @@ -45,7 +56,7 @@ extern "C" { * * \return Non-negative on success or negative on failure */ -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_thread_pool_init(unsigned int thread_count, hg_thread_pool_t **pool); /** @@ -55,7 +66,7 @@ hg_thread_pool_init(unsigned int thread_count, hg_thread_pool_t **pool); * * \return Non-negative on success or negative on failure */ -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_thread_pool_destroy(hg_thread_pool_t *pool); /** @@ -76,29 +87,16 @@ hg_thread_pool_post(hg_thread_pool_t *pool, struct hg_thread_work *work) { int ret = HG_UTIL_SUCCESS; - if (!pool) { - HG_UTIL_LOG_ERROR("Thread pool not initialized"); - ret = HG_UTIL_FAIL; - goto done; - } - - if (!work) { - HG_UTIL_LOG_ERROR("Thread work cannot be NULL"); - ret = HG_UTIL_FAIL; - goto done; - } + if (!pool || !work) + return HG_UTIL_FAIL; - if (!work->func) { - HG_UTIL_LOG_ERROR("Function pointer cannot be NULL"); - ret = HG_UTIL_FAIL; - goto done; - } + if (!work->func) + return HG_UTIL_FAIL; hg_thread_mutex_lock(&pool->mutex); /* Are we shutting down ? */ if (pool->shutdown) { - HG_UTIL_LOG_ERROR("Pool is shutting down"); ret = HG_UTIL_FAIL; goto unlock; } @@ -107,17 +105,13 @@ hg_thread_pool_post(hg_thread_pool_t *pool, struct hg_thread_work *work) HG_QUEUE_PUSH_TAIL(&pool->queue, work, entry); /* Wake up sleeping worker */ - if (pool->sleeping_worker_count) { - if (hg_thread_cond_signal(&pool->cond) != HG_UTIL_SUCCESS) { - HG_UTIL_LOG_ERROR("Cannot signal pool condition"); - ret = HG_UTIL_FAIL; - } - } + if (pool->sleeping_worker_count && + (hg_thread_cond_signal(&pool->cond) != HG_UTIL_SUCCESS)) + ret = HG_UTIL_FAIL; unlock: hg_thread_mutex_unlock(&pool->mutex); -done: return ret; } diff --git a/src/util/mercury_thread_rwlock.c b/src/util/mercury_thread_rwlock.c index 4390ad04..b7ffde42 100644 --- a/src/util/mercury_thread_rwlock.c +++ b/src/util/mercury_thread_rwlock.c @@ -52,28 +52,26 @@ int hg_thread_rwlock_init(hg_thread_rwlock_t *rwlock) { - int ret = HG_UTIL_SUCCESS; - #ifdef _WIN32 InitializeSRWLock(rwlock); #else - if (pthread_rwlock_init(rwlock, NULL)) ret = HG_UTIL_FAIL; + if (pthread_rwlock_init(rwlock, NULL)) + return HG_UTIL_FAIL; #endif - return ret; + return HG_UTIL_SUCCESS; } /*---------------------------------------------------------------------------*/ int hg_thread_rwlock_destroy(hg_thread_rwlock_t *rwlock) { - int ret = HG_UTIL_SUCCESS; - #ifdef _WIN32 /* nothing to do */ #else - if (pthread_rwlock_destroy(rwlock)) ret = HG_UTIL_FAIL; + if (pthread_rwlock_destroy(rwlock)) + return HG_UTIL_FAIL; #endif - return ret; + return HG_UTIL_SUCCESS; } diff --git a/src/util/mercury_thread_rwlock.h b/src/util/mercury_thread_rwlock.h index 7e72da5f..22985c89 100644 --- a/src/util/mercury_thread_rwlock.h +++ b/src/util/mercury_thread_rwlock.h @@ -50,12 +50,12 @@ #define MERCURY_THREAD_RWLOCK_H #include "mercury_util_config.h" + #ifdef _WIN32 -# include +# include typedef PSRWLOCK hg_thread_rwlock_t; #else -# include -# include +# include typedef pthread_rwlock_t hg_thread_rwlock_t; #endif @@ -70,7 +70,7 @@ extern "C" { * * \return Non-negative on success or negative on failure */ -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_thread_rwlock_init(hg_thread_rwlock_t *rwlock); /** @@ -80,7 +80,7 @@ hg_thread_rwlock_init(hg_thread_rwlock_t *rwlock); * * \return Non-negative on success or negative on failure */ -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_thread_rwlock_destroy(hg_thread_rwlock_t *rwlock); /** @@ -147,90 +147,86 @@ hg_thread_rwlock_release_wrlock(hg_thread_rwlock_t *rwlock); static HG_UTIL_INLINE int hg_thread_rwlock_rdlock(hg_thread_rwlock_t *rwlock) { - int ret = HG_UTIL_SUCCESS; - #ifdef _WIN32 AcquireSRWLockShared(rwlock); #else - if (pthread_rwlock_rdlock(rwlock)) ret = HG_UTIL_FAIL; + if (pthread_rwlock_rdlock(rwlock)) + return HG_UTIL_FAIL; #endif - return ret; + return HG_UTIL_SUCCESS; } /*---------------------------------------------------------------------------*/ static HG_UTIL_INLINE int hg_thread_rwlock_try_rdlock(hg_thread_rwlock_t *rwlock) { - int ret = HG_UTIL_SUCCESS; - #ifdef _WIN32 - if (TryAcquireSRWLockShared(rwlock) == 0) ret = HG_UTIL_FAIL; + if (TryAcquireSRWLockShared(rwlock) == 0) + return HG_UTIL_FAIL; #else - if (pthread_rwlock_tryrdlock(rwlock)) ret = HG_UTIL_FAIL; + if (pthread_rwlock_tryrdlock(rwlock)) + return HG_UTIL_FAIL; #endif - return ret; + return HG_UTIL_SUCCESS; } /*---------------------------------------------------------------------------*/ static HG_UTIL_INLINE int hg_thread_rwlock_release_rdlock(hg_thread_rwlock_t *rwlock) { - int ret = HG_UTIL_SUCCESS; - #ifdef _WIN32 ReleaseSRWLockShared(rwlock); #else - if (pthread_rwlock_unlock(rwlock)) ret = HG_UTIL_FAIL; + if (pthread_rwlock_unlock(rwlock)) + return HG_UTIL_FAIL; #endif - return ret; + return HG_UTIL_SUCCESS; } /*---------------------------------------------------------------------------*/ static HG_UTIL_INLINE int hg_thread_rwlock_wrlock(hg_thread_rwlock_t *rwlock) { - int ret = HG_UTIL_SUCCESS; - #ifdef _WIN32 ReleaseSRWLockExclusive(rwlock); #else - if (pthread_rwlock_wrlock(rwlock)) ret = HG_UTIL_FAIL; + if (pthread_rwlock_wrlock(rwlock)) + return HG_UTIL_FAIL; #endif - return ret; + return HG_UTIL_SUCCESS; } /*---------------------------------------------------------------------------*/ static HG_UTIL_INLINE int hg_thread_rwlock_try_wrlock(hg_thread_rwlock_t *rwlock) { - int ret = HG_UTIL_SUCCESS; - #ifdef _WIN32 - if (TryAcquireSRWLockExclusive(rwlock) == 0) ret = HG_UTIL_FAIL; + if (TryAcquireSRWLockExclusive(rwlock) == 0) + return HG_UTIL_FAIL; #else - if (pthread_rwlock_trywrlock(rwlock)) ret = HG_UTIL_FAIL; + if (pthread_rwlock_trywrlock(rwlock)) + return HG_UTIL_FAIL; #endif - return ret; + return HG_UTIL_SUCCESS; } /*---------------------------------------------------------------------------*/ static HG_UTIL_INLINE int hg_thread_rwlock_release_wrlock(hg_thread_rwlock_t *rwlock) { - int ret = HG_UTIL_SUCCESS; - #ifdef _WIN32 ReleaseSRWLockExclusive(rwlock); #else - if (pthread_rwlock_unlock(rwlock)) ret = HG_UTIL_FAIL; + if (pthread_rwlock_unlock(rwlock)) + return HG_UTIL_FAIL; #endif - return ret; + return HG_UTIL_SUCCESS; } #ifdef __cplusplus diff --git a/src/util/mercury_thread_spin.c b/src/util/mercury_thread_spin.c index 70968cf9..60ef1f6e 100644 --- a/src/util/mercury_thread_spin.c +++ b/src/util/mercury_thread_spin.c @@ -14,32 +14,34 @@ int hg_thread_spin_init(hg_thread_spin_t *lock) { - int ret = HG_UTIL_SUCCESS; - #if defined(_WIN32) *lock = 0; + + return HG_UTIL_SUCCESS; #elif defined(HG_UTIL_HAS_PTHREAD_SPINLOCK_T) - if (pthread_spin_init(lock, 0)) ret = HG_UTIL_FAIL; + if (pthread_spin_init(lock, 0)) + return HG_UTIL_FAIL; + + return HG_UTIL_SUCCESS; #else - ret = hg_thread_mutex_init(lock); + return hg_thread_mutex_init(lock); #endif - - return ret; } /*---------------------------------------------------------------------------*/ int hg_thread_spin_destroy(hg_thread_spin_t *lock) { - int ret = HG_UTIL_SUCCESS; - #if defined(_WIN32) (void) lock; + + return HG_UTIL_SUCCESS; #elif defined(HG_UTIL_HAS_PTHREAD_SPINLOCK_T) - if (pthread_spin_destroy(lock)) ret = HG_UTIL_FAIL; + if (pthread_spin_destroy(lock)) + return HG_UTIL_FAIL; + + return HG_UTIL_SUCCESS; #else - ret = hg_thread_mutex_destroy(lock); + return hg_thread_mutex_destroy(lock); #endif - - return ret; } diff --git a/src/util/mercury_thread_spin.h b/src/util/mercury_thread_spin.h index dd2e262c..661d0848 100644 --- a/src/util/mercury_thread_spin.h +++ b/src/util/mercury_thread_spin.h @@ -12,16 +12,16 @@ #define MERCURY_THREAD_SPIN_H #include "mercury_util_config.h" + #if defined(_WIN32) -# include +# include typedef volatile LONG hg_thread_spin_t; #elif defined(HG_UTIL_HAS_PTHREAD_SPINLOCK_T) -# include -# include +# include typedef pthread_spinlock_t hg_thread_spin_t; #else /* Default to hg_thread_mutex_t if pthread_spinlock_t is not supported */ -# include "mercury_thread_mutex.h" +# include "mercury_thread_mutex.h" typedef hg_thread_mutex_t hg_thread_spin_t; #endif @@ -36,7 +36,7 @@ extern "C" { * * \return Non-negative on success or negative on failure */ -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_thread_spin_init(hg_thread_spin_t *lock); /** @@ -46,7 +46,7 @@ hg_thread_spin_init(hg_thread_spin_t *lock); * * \return Non-negative on success or negative on failure */ -HG_UTIL_EXPORT int +HG_UTIL_PUBLIC int hg_thread_spin_destroy(hg_thread_spin_t *lock); /** @@ -83,8 +83,6 @@ hg_thread_spin_unlock(hg_thread_spin_t *lock); static HG_UTIL_INLINE int hg_thread_spin_lock(hg_thread_spin_t *lock) { - int ret = HG_UTIL_SUCCESS; - #if defined(_WIN32) while (InterlockedExchange(lock, EBUSY)) { /* Don't lock while waiting */ @@ -95,50 +93,50 @@ hg_thread_spin_lock(hg_thread_spin_t *lock) MemoryBarrier(); } } + return HG_UTIL_SUCCESS; #elif defined(HG_UTIL_HAS_PTHREAD_SPINLOCK_T) - if (pthread_spin_lock(lock)) ret = HG_UTIL_FAIL; + if (pthread_spin_lock(lock)) + return HG_UTIL_FAIL; + + return HG_UTIL_SUCCESS; #else - ret = hg_thread_mutex_lock(lock); + return hg_thread_mutex_lock(lock); #endif - - return ret; } /*---------------------------------------------------------------------------*/ static HG_UTIL_INLINE int hg_thread_spin_try_lock(hg_thread_spin_t *lock) { - int ret = HG_UTIL_SUCCESS; - #if defined(_WIN32) - ret = InterlockedExchange(lock, EBUSY); + return InterlockedExchange(lock, EBUSY); #elif defined(HG_UTIL_HAS_PTHREAD_SPINLOCK_T) - if (pthread_spin_trylock(lock)) ret = HG_UTIL_FAIL; + if (pthread_spin_trylock(lock)) + return HG_UTIL_FAIL; + + return HG_UTIL_SUCCESS; #else - ret = hg_thread_mutex_try_lock(lock); + return hg_thread_mutex_try_lock(lock); #endif - - return ret; } /*---------------------------------------------------------------------------*/ static HG_UTIL_INLINE int hg_thread_spin_unlock(hg_thread_spin_t *lock) { - int ret = HG_UTIL_SUCCESS; - #if defined(_WIN32) /* Compiler barrier. The store below acts with release semantics */ MemoryBarrier(); - *lock = 0; + + return HG_UTIL_SUCCESS; #elif defined(HG_UTIL_HAS_PTHREAD_SPINLOCK_T) - if (pthread_spin_unlock(lock)) ret = HG_UTIL_FAIL; + if (pthread_spin_unlock(lock)) + return HG_UTIL_FAIL; + return HG_UTIL_SUCCESS; #else - ret = hg_thread_mutex_unlock(lock); + return hg_thread_mutex_unlock(lock); #endif - - return ret; } #ifdef __cplusplus diff --git a/src/util/mercury_time.h b/src/util/mercury_time.h index 7055a35e..3493a9f4 100644 --- a/src/util/mercury_time.h +++ b/src/util/mercury_time.h @@ -12,34 +12,47 @@ #define MERCURY_TIME_H #include "mercury_util_config.h" + #if defined(_WIN32) -# include +# include #elif defined(HG_UTIL_HAS_CLOCK_MONOTONIC) -# if defined(HG_UTIL_HAS_TIME_H) && defined(HG_UTIL_HAS_CLOCK_GETTIME) -# include -# elif defined(__APPLE__) && defined(HG_UTIL_HAS_SYSTIME_H) -# include -# include -# else -# error "Not supported on this platform." -# endif +# if defined(HG_UTIL_HAS_TIME_H) && defined(HG_UTIL_HAS_CLOCK_GETTIME) +# include +# elif defined(__APPLE__) && defined(HG_UTIL_HAS_SYSTIME_H) +# include +# include +# else +# error "Not supported on this platform." +# endif #else -# include -# include -# if defined(HG_UTIL_HAS_SYSTIME_H) -# include -# else -# error "Not supported on this platform." -# endif +# include +# include +# if defined(HG_UTIL_HAS_SYSTIME_H) +# include +# else +# error "Not supported on this platform." +# endif #endif +/*************************************/ +/* Public Type and Struct Definition */ +/*************************************/ + typedef struct hg_time hg_time_t; -struct hg_time -{ + +struct hg_time { long tv_sec; long tv_usec; }; +/*****************/ +/* Public Macros */ +/*****************/ + +/*********************/ +/* Public Prototypes */ +/*********************/ + #ifdef __cplusplus extern "C" { #endif @@ -160,7 +173,6 @@ hg_time_get_current(hg_time_t *tv) static double freq_to_usec; static int initialized = 0; static BOOL use_perf_counter = 0; - int ret = HG_UTIL_SUCCESS; if (!tv) return HG_UTIL_FAIL; @@ -192,12 +204,12 @@ hg_time_get_current(hg_time_t *tv) tv->tv_sec = t.QuadPart / 1000000; tv->tv_usec = t.QuadPart % 1000000; - return ret; + return HG_UTIL_SUCCESS; } #elif defined(HG_UTIL_HAS_CLOCK_MONOTONIC) /*---------------------------------------------------------------------------*/ -# if defined(HG_UTIL_HAS_TIME_H) && defined(HG_UTIL_HAS_CLOCK_GETTIME) +# if defined(HG_UTIL_HAS_TIME_H) && defined(HG_UTIL_HAS_CLOCK_GETTIME) static HG_UTIL_INLINE int hg_time_get_current(hg_time_t *tv) { @@ -216,13 +228,12 @@ hg_time_get_current(hg_time_t *tv) } /*---------------------------------------------------------------------------*/ -# elif defined(__APPLE__) && defined(HG_UTIL_HAS_SYSTIME_H) +# elif defined(__APPLE__) && defined(HG_UTIL_HAS_SYSTIME_H) static HG_UTIL_INLINE int hg_time_get_current(hg_time_t *tv) { static uint64_t monotonic_timebase_factor = 0; uint64_t monotonic_nsec; - int ret = HG_UTIL_SUCCESS; if (!tv) return HG_UTIL_FAIL; @@ -234,16 +245,16 @@ hg_time_get_current(hg_time_t *tv) monotonic_timebase_factor = timebase_info.numer / timebase_info.denom; } monotonic_nsec = (mach_absolute_time() * monotonic_timebase_factor); - tv->tv_sec = (long) (monotonic_nsec / 1000000000); + tv->tv_sec = (long) (monotonic_nsec / 1000000000); tv->tv_usec = (long) ((monotonic_nsec - (uint64_t) tv->tv_sec) / 1000); - return ret; + return HG_UTIL_SUCCESS; } -# endif +# endif #else /*---------------------------------------------------------------------------*/ -# if defined(HG_UTIL_HAS_SYSTIME_H) +# if defined(HG_UTIL_HAS_SYSTIME_H) static HG_UTIL_INLINE int hg_time_get_current(hg_time_t *tv) { @@ -255,7 +266,7 @@ hg_time_get_current(hg_time_t *tv) return HG_UTIL_SUCCESS; } -# endif +# endif #endif /*---------------------------------------------------------------------------*/ static HG_UTIL_INLINE double @@ -281,7 +292,7 @@ static HG_UTIL_INLINE int hg_time_less(hg_time_t in1, hg_time_t in2) { return ((in1.tv_sec < in2.tv_sec) || - ((in1.tv_sec == in2.tv_sec) && (in1.tv_usec < in2.tv_usec))); + ((in1.tv_sec == in2.tv_sec) && (in1.tv_usec < in2.tv_usec))); } /*---------------------------------------------------------------------------*/ @@ -292,7 +303,7 @@ hg_time_add(hg_time_t in1, hg_time_t in2) out.tv_sec = in1.tv_sec + in2.tv_sec; out.tv_usec = in1.tv_usec + in2.tv_usec; - if(out.tv_usec > 1000000) { + if (out.tv_usec > 1000000) { out.tv_usec -= 1000000; out.tv_sec += 1; } @@ -308,7 +319,7 @@ hg_time_subtract(hg_time_t in1, hg_time_t in2) out.tv_sec = in1.tv_sec - in2.tv_sec; out.tv_usec = in1.tv_usec - in2.tv_usec; - if(out.tv_usec < 0) { + if (out.tv_usec < 0) { out.tv_usec += 1000000; out.tv_sec -= 1; } @@ -320,10 +331,8 @@ hg_time_subtract(hg_time_t in1, hg_time_t in2) static HG_UTIL_INLINE int hg_time_sleep(const hg_time_t rqt) { - int ret = HG_UTIL_SUCCESS; - #ifdef _WIN32 - DWORD dwMilliseconds = (DWORD) (hg_time_to_double(rqt) / 1000); + DWORD dwMilliseconds = (DWORD)(hg_time_to_double(rqt) / 1000); Sleep(dwMilliseconds); #elif defined(HG_UTIL_HAS_CLOCK_MONOTONIC) @@ -332,20 +341,17 @@ hg_time_sleep(const hg_time_t rqt) rqtp.tv_sec = rqt.tv_sec; rqtp.tv_nsec = rqt.tv_usec * 1000; - if (nanosleep(&rqtp, NULL)) { - ret = HG_UTIL_FAIL; - return ret; - } + if (nanosleep(&rqtp, NULL)) + return HG_UTIL_FAIL; #else - useconds_t usec = (useconds_t) rqt.tv_sec * 1000000 + (useconds_t) rqt.tv_usec; + useconds_t usec = + (useconds_t) rqt.tv_sec * 1000000 + (useconds_t) rqt.tv_usec; - if (usleep(usec)) { - ret = HG_UTIL_FAIL; - return ret; - } + if (usleep(usec)) + return HG_UTIL_FAIL; #endif - return ret; + return HG_UTIL_SUCCESS; } /*---------------------------------------------------------------------------*/ @@ -353,8 +359,7 @@ hg_time_sleep(const hg_time_t rqt) static HG_UTIL_INLINE char * hg_time_stamp(void) { - char *ret = NULL; - static char buf[HG_UTIL_STAMP_MAX]; + static char buf[HG_UTIL_STAMP_MAX] = {'\0'}; #if defined(_WIN32) /* TODO not implemented */ @@ -364,18 +369,11 @@ hg_time_stamp(void) t = time(NULL); local_time = localtime(&t); - if (local_time == NULL) { - ret = NULL; - return ret; - } - - if (strftime(buf, HG_UTIL_STAMP_MAX, "%a, %d %b %Y %T %Z", - local_time) == 0) { - ret = NULL; - return ret; - } + if (local_time == NULL) + return NULL; - ret = buf; + if (strftime(buf, HG_UTIL_STAMP_MAX, "%a, %d %b %Y %T %Z", local_time) == 0) + return NULL; #else struct timeval tv; struct timezone tz; @@ -384,17 +382,17 @@ hg_time_stamp(void) gettimeofday(&tv, &tz); days = (unsigned long) tv.tv_sec / (3600 * 24); hours = ((unsigned long) tv.tv_sec - days * 24 * 3600) / 3600; - minutes = ((unsigned long) tv.tv_sec - days * 24 * 3600 - hours * 3600) / 60; - seconds = (unsigned long) tv.tv_sec - days * 24 * 3600 - hours * 3600 - minutes * 60; + minutes = + ((unsigned long) tv.tv_sec - days * 24 * 3600 - hours * 3600) / 60; + seconds = (unsigned long) tv.tv_sec - days * 24 * 3600 - hours * 3600 - + minutes * 60; hours -= (unsigned long) tz.tz_minuteswest / 60; - snprintf(buf, HG_UTIL_STAMP_MAX, "%02lu:%02lu:%02lu (GMT-%d)", - hours, minutes, seconds, tz.tz_minuteswest / 60); - - ret = buf; + snprintf(buf, HG_UTIL_STAMP_MAX, "%02lu:%02lu:%02lu (GMT-%d)", hours, + minutes, seconds, tz.tz_minuteswest / 60); #endif - return ret; + return buf; } #ifdef __cplusplus diff --git a/src/util/mercury_util_config.h.in b/src/util/mercury_util_config.h.in index 034df921..57b3d029 100644 --- a/src/util/mercury_util_config.h.in +++ b/src/util/mercury_util_config.h.in @@ -13,113 +13,129 @@ #ifndef MERCURY_UTIL_CONFIG_H #define MERCURY_UTIL_CONFIG_H -/* Import/export declarations */ +/*************************************/ +/* Public Type and Struct Definition */ +/*************************************/ + +/* Type definitions */ +#ifdef _WIN32 +typedef signed __int64 hg_util_int64_t; +typedef signed __int32 hg_util_int32_t; +typedef signed __int16 hg_util_int16_t; +typedef signed __int8 hg_util_int8_t; +typedef unsigned __int64 hg_util_uint64_t; +typedef unsigned __int32 hg_util_uint32_t; +typedef unsigned __int16 hg_util_uint16_t; +typedef unsigned __int8 hg_util_uint8_t; +#else +# include +# include +typedef int64_t hg_util_int64_t; +typedef int32_t hg_util_int32_t; +typedef int16_t hg_util_int16_t; +typedef int8_t hg_util_int8_t; +typedef uint64_t hg_util_uint64_t; +typedef uint32_t hg_util_uint32_t; +typedef uint16_t hg_util_uint16_t; +typedef uint8_t hg_util_uint8_t; +#endif +typedef hg_util_uint8_t hg_util_bool_t; +typedef hg_util_uint64_t hg_util_ptr_t; + +/* True / false */ +#define HG_UTIL_TRUE 1 +#define HG_UTIL_FALSE 0 + +/* Return codes */ +#define HG_UTIL_SUCCESS 0 +#define HG_UTIL_FAIL -1 + +/*****************/ +/* Public Macros */ +/*****************/ + +/* Visibility of symbols */ #if defined(_WIN32) - #define HG_UTIL_ABI_IMPORT __declspec(dllimport) - #define HG_UTIL_ABI_EXPORT __declspec(dllexport) - #define HG_UTIL_ABI_HIDDEN +# define HG_UTIL_ABI_IMPORT __declspec(dllimport) +# define HG_UTIL_ABI_EXPORT __declspec(dllexport) +# define HG_UTIL_ABI_HIDDEN #elif defined(__GNUC__) && (__GNUC__ >= 4) - #define HG_UTIL_ABI_IMPORT __attribute__((visibility("default"))) - #define HG_UTIL_ABI_EXPORT __attribute__((visibility("default"))) - #define HG_UTIL_ABI_HIDDEN __attribute__((visibility("hidden"))) +# define HG_UTIL_ABI_IMPORT __attribute__((visibility("default"))) +# define HG_UTIL_ABI_EXPORT __attribute__((visibility("default"))) +# define HG_UTIL_ABI_HIDDEN __attribute__((visibility("hidden"))) #else - #define HG_UTIL_ABI_IMPORT - #define HG_UTIL_ABI_EXPORT - #define HG_UTIL_ABI_HIDDEN +# define HG_UTIL_ABI_IMPORT +# define HG_UTIL_ABI_EXPORT +# define HG_UTIL_ABI_HIDDEN #endif -/* Define if has pthread_spinlock_t type */ -#cmakedefine HG_UTIL_HAS_PTHREAD_SPINLOCK_T +/* Inline macro */ +#ifdef _WIN32 +# define HG_UTIL_INLINE __inline +#else +# define HG_UTIL_INLINE __inline__ +#endif -/* Define if has PTHREAD_MUTEX_ADAPTIVE_NP */ -#cmakedefine HG_UTIL_HAS_PTHREAD_MUTEX_ADAPTIVE_NP +/* Shared libraries */ +#cmakedefine HG_UTIL_BUILD_SHARED_LIBS +#ifdef HG_UTIL_BUILD_SHARED_LIBS +# ifdef mercury_util_EXPORTS +# define HG_UTIL_PUBLIC HG_UTIL_ABI_EXPORT +# else +# define HG_UTIL_PUBLIC HG_UTIL_ABI_IMPORT +# endif +# define HG_UTIL_PRIVATE HG_UTIL_ABI_HIDDEN +#else +# define HG_UTIL_PUBLIC +# define HG_UTIL_PRIVATE +#endif -/* Define if has 'pthread_condattr_setclock()' */ -#cmakedefine HG_UTIL_HAS_PTHREAD_CONDATTR_SETCLOCK +/* Define if has 'clock_gettime()' */ +#cmakedefine HG_UTIL_HAS_CLOCK_GETTIME + +/* Define if has CLOCK_MONOTONIC */ +#cmakedefine HG_UTIL_HAS_CLOCK_MONOTONIC + +/* Define if has eventfd_t type */ +#cmakedefine HG_UTIL_HAS_EVENTFD_T + +/* Define if has colored output */ +#cmakedefine HG_UTIL_HAS_LOG_COLOR /* Define if has */ #cmakedefine HG_UTIL_HAS_OPA_PRIMITIVES_H -/* Define if has */ -#cmakedefine HG_UTIL_HAS_STDATOMIC_H +/* Define if has 'pthread_condattr_setclock()' */ +#cmakedefine HG_UTIL_HAS_PTHREAD_CONDATTR_SETCLOCK -/* Define if has CLOCK_MONOTONIC */ -#cmakedefine HG_UTIL_HAS_CLOCK_MONOTONIC +/* Define if has PTHREAD_MUTEX_ADAPTIVE_NP */ +#cmakedefine HG_UTIL_HAS_PTHREAD_MUTEX_ADAPTIVE_NP -/* Define if has */ -#cmakedefine HG_UTIL_HAS_TIME_H +/* Define if has pthread_spinlock_t type */ +#cmakedefine HG_UTIL_HAS_PTHREAD_SPINLOCK_T -/* Define if has 'clock_gettime()' */ -#cmakedefine HG_UTIL_HAS_CLOCK_GETTIME +/* Define if has */ +#cmakedefine HG_UTIL_HAS_STDATOMIC_H -/* Define if has */ -#cmakedefine HG_UTIL_HAS_SYSTIME_H +/* Define type size of atomic_long */ +#cmakedefine HG_UTIL_ATOMIC_LONG_WIDTH @HG_UTIL_ATOMIC_LONG_WIDTH@ /* Define if has */ #cmakedefine HG_UTIL_HAS_SYSEPOLL_H +/* Define if has */ +#cmakedefine HG_UTIL_HAS_SYSEVENT_H + /* Define if has */ #cmakedefine HG_UTIL_HAS_SYSEVENTFD_H -/* Define if has eventfd_t type */ -#cmakedefine HG_UTIL_HAS_EVENTFD_T +/* Define if has */ +#cmakedefine HG_UTIL_HAS_SYSTIME_H -/* Define if has */ -#cmakedefine HG_UTIL_HAS_SYSEVENT_H +/* Define if has */ +#cmakedefine HG_UTIL_HAS_TIME_H /* Define if has verbose error */ #cmakedefine HG_UTIL_HAS_VERBOSE_ERROR -/* Define if build shared libraries */ -#cmakedefine HG_UTIL_BUILD_SHARED_LIBS - -/* Define export declaration */ -#ifdef HG_UTIL_BUILD_SHARED_LIBS - #ifdef mercury_util_EXPORTS - #define HG_UTIL_EXPORT HG_UTIL_ABI_EXPORT - #else - #define HG_UTIL_EXPORT HG_UTIL_ABI_IMPORT - #endif -#else - #define HG_UTIL_EXPORT -#endif - -/* Standard types */ -#ifdef _WIN32 - typedef signed __int64 hg_util_int64_t; - typedef signed __int32 hg_util_int32_t; - typedef signed __int16 hg_util_int16_t; - typedef signed __int8 hg_util_int8_t; - typedef unsigned __int64 hg_util_uint64_t; - typedef unsigned __int32 hg_util_uint32_t; - typedef unsigned __int16 hg_util_uint16_t; - typedef unsigned __int8 hg_util_uint8_t; -#else - #include - #include - typedef int64_t hg_util_int64_t; - typedef int32_t hg_util_int32_t; - typedef int16_t hg_util_int16_t; - typedef int8_t hg_util_int8_t; - typedef uint64_t hg_util_uint64_t; - typedef uint32_t hg_util_uint32_t; - typedef uint16_t hg_util_uint16_t; - typedef uint8_t hg_util_uint8_t; -#endif -typedef hg_util_uint8_t hg_util_bool_t; -typedef hg_util_uint64_t hg_util_ptr_t; - -/* Inline declarations */ -#ifdef _WIN32 - #define HG_UTIL_INLINE __inline -#else - #define HG_UTIL_INLINE __inline__ -#endif - -/* Return codes */ -#define HG_UTIL_SUCCESS 0 -#define HG_UTIL_FAIL -1 -#define HG_UTIL_TRUE 1 -#define HG_UTIL_FALSE 0 - #endif /* MERCURY_UTIL_CONFIG_H */ diff --git a/src/util/mercury_util_error.c b/src/util/mercury_util_error.c new file mode 100644 index 00000000..0280c888 --- /dev/null +++ b/src/util/mercury_util_error.c @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2013-2019 Argonne National Laboratory, Department of Energy, + * UChicago Argonne, LLC and The HDF Group. + * All rights reserved. + * + * The full copyright notice, including terms governing use, modification, + * and redistribution, is contained in the COPYING file that can be + * found at the root of the source code distribution tree. + */ + +#include "mercury_util_error.h" + +/*******************/ +/* Local Variables */ +/*******************/ + +/* Default error log mask */ +#ifdef HG_UTIL_HAS_VERBOSE_ERROR +unsigned int HG_UTIL_LOG_MASK = HG_LOG_TYPE_ERROR | HG_LOG_TYPE_WARNING; +#endif \ No newline at end of file diff --git a/src/util/mercury_util_error.h b/src/util/mercury_util_error.h index f22fad0b..6c12bda1 100644 --- a/src/util/mercury_util_error.h +++ b/src/util/mercury_util_error.h @@ -15,18 +15,90 @@ /* Default error macro */ #ifdef HG_UTIL_HAS_VERBOSE_ERROR - #include - #define HG_UTIL_LOG_MODULE_NAME "HG Util" - #define HG_UTIL_LOG_ERROR(...) \ - HG_LOG_WRITE_ERROR(HG_UTIL_LOG_MODULE_NAME, __VA_ARGS__) - #define HG_UTIL_LOG_DEBUG(...) \ - HG_LOG_WRITE_DEBUG(HG_UTIL_LOG_MODULE_NAME, __VA_ARGS__) - #define HG_UTIL_LOG_WARNING(...) \ - HG_LOG_WRITE_WARNING(HG_UTIL_LOG_MODULE_NAME, __VA_ARGS__) +# include +# define HG_UTIL_LOG_MASK hg_util_log_mask +/* Log mask will be initialized in init routine */ +extern HG_UTIL_PRIVATE unsigned int HG_UTIL_LOG_MASK; +# define HG_UTIL_LOG_MODULE_NAME "HG Util" +# define HG_UTIL_LOG_ERROR(...) \ + do { \ + if (HG_UTIL_LOG_MASK & HG_LOG_TYPE_ERROR) \ + HG_LOG_WRITE_ERROR(HG_UTIL_LOG_MODULE_NAME, __VA_ARGS__); \ + } while (0) +# ifdef HG_UTIL_HAS_DEBUG +# define HG_UTIL_LOG_DEBUG(...) \ + do { \ + if (HG_UTIL_LOG_MASK & HG_LOG_TYPE_DEBUG) \ + HG_LOG_WRITE_DEBUG(HG_UTIL_LOG_MODULE_NAME, __VA_ARGS__); \ + } while (0) +# else +# define HG_UTIL_LOG_DEBUG(...) (void) 0 +# endif +# define HG_UTIL_LOG_WARNING(...) \ + do { \ + if (HG_UTIL_LOG_MASK & HG_LOG_TYPE_WARNING) \ + HG_LOG_WRITE_WARNING(HG_UTIL_LOG_MODULE_NAME, __VA_ARGS__); \ + } while (0) #else - #define HG_UTIL_LOG_ERROR(...) (void)0 - #define HG_UTIL_LOG_DEBUG(...) (void)0 - #define HG_UTIL_LOG_WARNING(...) (void)0 +# define HG_UTIL_LOG_ERROR(...) (void) 0 +# define HG_UTIL_LOG_DEBUG(...) (void) 0 +# define HG_UTIL_LOG_WARNING(...) (void) 0 #endif +/* Branch predictor hints */ +#ifndef _WIN32 +# define likely(x) __builtin_expect(!!(x), 1) +# define unlikely(x) __builtin_expect(!!(x), 0) +#else +# define likely(x) (x) +# define unlikely(x) (x) +#endif + +/* Error macros */ +#define HG_UTIL_GOTO_DONE(label, ret, ret_val) \ + do { \ + ret = ret_val; \ + goto label; \ + } while (0) + +#define HG_UTIL_GOTO_ERROR(label, ret, err_val, ...) \ + do { \ + HG_UTIL_LOG_ERROR(__VA_ARGS__); \ + ret = err_val; \ + goto label; \ + } while (0) + +/* Check for cond, set ret to err_val and goto label */ +#define HG_UTIL_CHECK_ERROR(cond, label, ret, err_val, ...) \ + do { \ + if (unlikely(cond)) { \ + HG_UTIL_LOG_ERROR(__VA_ARGS__); \ + ret = err_val; \ + goto label; \ + } \ + } while (0) + +#define HG_UTIL_CHECK_ERROR_NORET(cond, label, ...) \ + do { \ + if (unlikely(cond)) { \ + HG_UTIL_LOG_ERROR(__VA_ARGS__); \ + goto label; \ + } \ + } while (0) + +#define HG_UTIL_CHECK_ERROR_DONE(cond, ...) \ + do { \ + if (unlikely(cond)) { \ + HG_UTIL_LOG_ERROR(__VA_ARGS__); \ + } \ + } while (0) + +/* Check for cond and print warning */ +#define HG_UTIL_CHECK_WARNING(cond, ...) \ + do { \ + if (unlikely(cond)) { \ + HG_UTIL_LOG_WARNING(__VA_ARGS__); \ + } \ + } while (0) + #endif /* MERCURY_UTIL_ERROR_H */ From c716b465caa4ef28335c989ad7654fdf986a3932 Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Thu, 14 May 2020 18:12:37 -0500 Subject: [PATCH 07/30] HG Core: update hg_core_poll_wait() after hg_poll_wait() changes --- src/mercury_core.c | 45 +++++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/src/mercury_core.c b/src/mercury_core.c index 446440b0..cd4587ae 100644 --- a/src/mercury_core.c +++ b/src/mercury_core.c @@ -41,6 +41,7 @@ #define HG_CORE_ATOMIC_QUEUE_SIZE 1024 #define HG_CORE_PENDING_INCR 256 #define HG_CORE_CLEANUP_TIMEOUT 1000 +#define HG_CORE_MAX_EVENTS 16 #define HG_CORE_MAX_TRIGGER_COUNT 1 #ifdef HG_HAS_SM_ROUTING # define HG_CORE_UUID_MAX_LEN 36 @@ -679,7 +680,7 @@ static HG_INLINE int hg_core_completion_queue_notify_cb( void *arg, int error, - hg_util_bool_t *progressed + struct hg_poll_event *event ); #endif @@ -690,7 +691,7 @@ static int hg_core_progress_na_cb( void *arg, int error, - hg_util_bool_t *progressed + struct hg_poll_event *event ); #ifdef HG_HAS_SM_ROUTING @@ -701,7 +702,7 @@ static int hg_core_progress_na_sm_cb( void *arg, int error, - hg_util_bool_t *progressed + struct hg_poll_event *event ); #endif @@ -810,7 +811,7 @@ hg_core_get_sm_uuid(uuid_t *sm_uuid) char uuid_str[HG_CORE_UUID_MAX_LEN + 1]; FILE *uuid_config; uuid_t new_uuid; - na_return_t ret = NA_SUCCESS; + hg_return_t ret = HG_SUCCESS; uuid_config = fopen(sm_path, "r"); if (!uuid_config) { @@ -2851,7 +2852,7 @@ hg_core_reset_post(struct hg_core_private_handle *hg_core_handle) #ifdef HG_HAS_SELF_FORWARD static HG_INLINE int hg_core_completion_queue_notify_cb(void *arg, - int HG_UNUSED error, hg_util_bool_t *progressed) + int HG_UNUSED error, struct hg_poll_event *event) { struct hg_core_private_context *context = (struct hg_core_private_context *) arg; @@ -2867,11 +2868,11 @@ hg_core_completion_queue_notify_cb(void *arg, if (notified || !hg_atomic_queue_is_empty(context->completion_queue) || hg_atomic_get32(&context->backfill_queue_count)) { - *progressed = HG_UTIL_TRUE; /* Progressed */ + event->progressed = HG_UTIL_TRUE; /* Progressed */ goto done; } - *progressed = HG_UTIL_FALSE; + event->progressed = HG_UTIL_FALSE; done: return rc; @@ -2881,7 +2882,7 @@ hg_core_completion_queue_notify_cb(void *arg, /*---------------------------------------------------------------------------*/ static int hg_core_progress_na_cb(void *arg, int HG_UNUSED error, - hg_util_bool_t *progressed) + struct hg_poll_event *event) { struct hg_core_private_context *context = (struct hg_core_private_context *) arg; @@ -2896,7 +2897,7 @@ hg_core_progress_na_cb(void *arg, int HG_UNUSED error, context->core_context.na_context, 0); if (na_ret == NA_TIMEOUT) { /* Nothing progressed */ - *progressed = HG_UTIL_FALSE; + event->progressed = HG_UTIL_FALSE; goto done; } else HG_CHECK_ERROR(na_ret != NA_SUCCESS, done, rc, HG_UTIL_FAIL, @@ -2924,11 +2925,11 @@ hg_core_progress_na_cb(void *arg, int HG_UNUSED error, if (!completed_count && hg_atomic_queue_is_empty(context->completion_queue) && !hg_atomic_get32(&context->backfill_queue_count)) { /* Nothing progressed */ - *progressed = HG_UTIL_FALSE; + event->progressed = HG_UTIL_FALSE; goto done; } - *progressed = HG_UTIL_TRUE; + event->progressed = HG_UTIL_TRUE; done: return rc; @@ -2938,7 +2939,7 @@ hg_core_progress_na_cb(void *arg, int HG_UNUSED error, #ifdef HG_HAS_SM_ROUTING static int hg_core_progress_na_sm_cb(void *arg, int HG_UNUSED error, - hg_util_bool_t *progressed) + struct hg_poll_event *event) { struct hg_core_private_context *context = (struct hg_core_private_context *) arg; @@ -2953,7 +2954,7 @@ hg_core_progress_na_sm_cb(void *arg, int HG_UNUSED error, context->core_context.na_sm_context, 0); if (na_ret == NA_TIMEOUT) { /* Nothing progressed */ - *progressed = HG_UTIL_FALSE; + event->progressed = HG_UTIL_FALSE; goto done; } else HG_CHECK_ERROR(na_ret != NA_SUCCESS, done, rc, HG_UTIL_FAIL, @@ -2982,11 +2983,11 @@ hg_core_progress_na_sm_cb(void *arg, int HG_UNUSED error, if (!completed_count && hg_atomic_queue_is_empty(context->completion_queue) && !hg_atomic_get32(&context->backfill_queue_count)) { /* Nothing progressed */ - *progressed = HG_UTIL_FALSE; + event->progressed = HG_UTIL_FALSE; goto done; } - *progressed = HG_UTIL_TRUE; + event->progressed = HG_UTIL_TRUE; done: return rc; @@ -3114,24 +3115,28 @@ hg_core_progress_poll(struct hg_core_private_context *context, do { hg_time_t t1, t2; - hg_util_bool_t progressed; + struct hg_poll_event events[HG_CORE_MAX_EVENTS] = {0}; unsigned int poll_timeout = (HG_CORE_CONTEXT_CLASS(context)->progress_mode & NA_NO_BLOCK) ? 0 : (unsigned int) (remaining * 1000.0); + unsigned int nevents, i; int rc; if (timeout) hg_time_get_current(&t1); /* Will call hg_core_poll_try_wait_cb if timeout is not 0 */ - rc = hg_poll_wait(context->poll_set, poll_timeout, &progressed); + rc = hg_poll_wait(context->poll_set, poll_timeout, HG_CORE_MAX_EVENTS, + events, &nevents); HG_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, HG_PROTOCOL_ERROR, "hg_poll_wait() failed"); /* We progressed, return success */ - if (progressed) { - ret = HG_SUCCESS; - break; + for (i = 0; i < nevents; i++) { + if (events[i].progressed) { + ret = HG_SUCCESS; + goto done; + } } if (timeout) { From d3cbc618ce15e69baa244fa81845d19f86fb1ec8 Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Thu, 14 May 2020 18:23:52 -0500 Subject: [PATCH 08/30] NA: remove callback from NA_Addr_lookup() Update BMI/MPI/CCI/OFI plugins accordingly Remove NA_CB_LOOKUP info type --- src/na/na.c | 60 ++++++++------------------------------- src/na/na.h | 42 +-------------------------- src/na/na_bmi.c | 57 ++++--------------------------------- src/na/na_cci.c | 72 ++++------------------------------------------- src/na/na_mpi.c | 46 +++--------------------------- src/na/na_ofi.c | 15 ++++++---- src/na/na_types.h | 16 +++++------ 7 files changed, 44 insertions(+), 264 deletions(-) diff --git a/src/na/na.c b/src/na/na.c index 0d919f0e..d0d2995a 100644 --- a/src/na/na.c +++ b/src/na/na.c @@ -362,6 +362,14 @@ NA_Cleanup(void) unsigned int plugin_count = sizeof(na_class_table) / sizeof(na_class_table[0]) - 1; unsigned int i; +#ifdef NA_HAS_VERBOSE_ERROR + const char *log_level = NULL; + + /* Set log level */ + log_level = getenv("HG_NA_LOG_LEVEL"); + if (log_level && (strcmp(log_level, "debug") == 0)) + NA_LOG_MASK |= HG_LOG_TYPE_DEBUG; +#endif for (i = 0; i < plugin_count; i++) { if (!na_class_table[i]->cleanup) @@ -524,48 +532,7 @@ NA_Op_destroy(na_class_t *na_class, na_op_id_t op_id) /*---------------------------------------------------------------------------*/ na_return_t -NA_Addr_lookup(na_class_t *na_class, na_context_t *context, na_cb_t callback, - void *arg, const char *name, na_op_id_t *op_id) -{ - char *name_string = NULL; - char *short_name = NULL; - na_return_t ret = NA_SUCCESS; - - NA_CHECK_ERROR(na_class == NULL, done, ret, NA_INVALID_ARG, - "NULL NA class"); - NA_CHECK_ERROR(context == NULL, done, ret, NA_INVALID_ARG, - "NULL context"); - NA_CHECK_ERROR(name == NULL, done, ret, NA_INVALID_ARG, - "Lookup name is NULL"); - - NA_CHECK_ERROR(na_class->ops == NULL, done, ret, NA_INVALID_ARG, - "NULL NA class ops"); - NA_CHECK_ERROR(na_class->ops->addr_lookup == NULL, done, ret, - NA_OPNOTSUPPORTED, "addr_lookup plugin callback is not defined"); - - /* Copy name and work from that */ - name_string = strdup(name); - NA_CHECK_ERROR(name_string == NULL, done, ret, NA_NOMEM, - "Could not duplicate string"); - - /* If NA class name was specified, we can remove the name here: - * ie. bmi+tcp://hostname:port -> tcp://hostname:port */ - if (strstr(name_string, NA_CLASS_DELIMITER) != NULL) - strtok_r(name_string, NA_CLASS_DELIMITER, &short_name); - else - short_name = name_string; - - ret = na_class->ops->addr_lookup(na_class, context, callback, arg, - short_name, op_id); - -done: - free(name_string); - return ret; -} - -/*---------------------------------------------------------------------------*/ -na_return_t -NA_Addr_lookup2(na_class_t *na_class, const char *name, na_addr_t *addr) +NA_Addr_lookup(na_class_t *na_class, const char *name, na_addr_t *addr) { char *name_string = NULL; char *short_name = NULL; @@ -580,11 +547,8 @@ NA_Addr_lookup2(na_class_t *na_class, const char *name, na_addr_t *addr) NA_CHECK_ERROR(na_class->ops == NULL, done, ret, NA_INVALID_ARG, "NULL NA class ops"); - if (!na_class->ops->addr_lookup2) - /* Until we switch to new lookup, exit if no callback */ - goto done; -// NA_CHECK_ERROR(na_class->ops->addr_lookup2 == NULL, done, ret, -// NA_PROTOCOL_ERROR, "addr_lookup2 plugin callback is not defined"); + NA_CHECK_ERROR(na_class->ops->addr_lookup == NULL, done, ret, + NA_PROTOCOL_ERROR, "addr_lookup2 plugin callback is not defined"); /* Copy name and work from that */ name_string = strdup(name); @@ -598,7 +562,7 @@ NA_Addr_lookup2(na_class_t *na_class, const char *name, na_addr_t *addr) else short_name = name_string; - ret = na_class->ops->addr_lookup2(na_class, short_name, addr); + ret = na_class->ops->addr_lookup(na_class, short_name, addr); done: free(name_string); diff --git a/src/na/na.h b/src/na/na.h index 4849ea71..b0b34b03 100644 --- a/src/na/na.h +++ b/src/na/na.h @@ -201,41 +201,10 @@ NA_Op_destroy( na_op_id_t op_id ); -/** - * Lookup an addr from a peer address/name. Addresses need to be - * freed by calling NA_Addr_free(). After completion, user callback is placed - * into a completion queue and can be triggered using NA_Trigger(). - * - * In the case where op_id is not NA_OP_ID_IGNORE and *op_id is NA_OP_ID_NULL, - * a new operation ID will be internally created and returned. Users may also - * manually create an operation ID through NA_Op_create() and pass it through - * op_id for future use and prevent multiple ID creation. - * - * \param na_class [IN/OUT] pointer to NA class - * \param context [IN/OUT] pointer to context of execution - * \param callback [IN] pointer to function callback - * \param arg [IN] pointer to data passed to callback - * \param name [IN] lookup name - * \param op_id [IN/OUT] pointer to operation ID - * - * \return NA_SUCCESS or corresponding NA error code - */ -NA_PUBLIC na_return_t -NA_Addr_lookup( - na_class_t *na_class, - na_context_t *context, - na_cb_t callback, - void *arg, - const char *name, - na_op_id_t *op_id - ); - /** * Lookup an addr from a peer address/name. Addresses need to be * freed by calling NA_Addr_free(). * - * \remark This is the immediate version of NA_Addr_lookup(). - * * \param na_class [IN/OUT] pointer to NA class * \param name [IN] lookup name * \param addr [OUT] pointer to abstract address @@ -243,7 +212,7 @@ NA_Addr_lookup( * \return NA_SUCCESS or corresponding NA error code */ NA_PUBLIC na_return_t -NA_Addr_lookup2( +NA_Addr_lookup( na_class_t *na_class, const char *name, na_addr_t *addr @@ -1164,15 +1133,6 @@ struct na_class_ops { ); na_return_t (*addr_lookup)( - na_class_t *na_class, - na_context_t *context, - na_cb_t callback, - void *arg, - const char *name, - na_op_id_t *op_id - ); - na_return_t - (*addr_lookup2)( na_class_t *na_class, const char *name, na_addr_t *addr diff --git a/src/na/na_bmi.c b/src/na/na_bmi.c index 95e531f5..338313aa 100644 --- a/src/na/na_bmi.c +++ b/src/na/na_bmi.c @@ -39,12 +39,12 @@ #define NA_BMI_EXPECTED_SIZE NA_BMI_UNEXPECTED_SIZE /* Max tag */ -#define NA_BMI_MAX_TAG (NA_TAG_UB >> 2) +#define NA_BMI_MAX_TAG (NA_TAG_MAX >> 2) /* Default tag used for one-sided over two-sided */ #define NA_BMI_RMA_REQUEST_TAG (NA_BMI_MAX_TAG + 1) #define NA_BMI_RMA_TAG (NA_BMI_RMA_REQUEST_TAG + 1) -#define NA_BMI_MAX_RMA_TAG (NA_TAG_UB >> 1) +#define NA_BMI_MAX_RMA_TAG (NA_TAG_MAX >> 1) #define NA_BMI_CLASS(na_class) \ ((struct na_bmi_class *)(na_class->plugin_class)) @@ -144,7 +144,6 @@ struct na_bmi_op_id { hg_atomic_int32_t completed; /* Operation completed */ uint64_t cancel; union { - struct na_bmi_info_lookup lookup; struct na_bmi_info_send_unexpected send_unexpected; struct na_bmi_info_recv_unexpected recv_unexpected; struct na_bmi_info_send_expected send_expected; @@ -223,11 +222,8 @@ na_bmi_op_destroy( static na_return_t na_bmi_addr_lookup( na_class_t *na_class, - na_context_t *context, - na_cb_t callback, - void *arg, const char *name, - na_op_id_t *op_id + na_addr_t *addr ); /* addr_free */ @@ -526,7 +522,6 @@ const struct na_class_ops NA_PLUGIN_OPS(bmi) = { na_bmi_op_create, /* op_create */ na_bmi_op_destroy, /* op_destroy */ na_bmi_addr_lookup, /* addr_lookup */ - NULL, /* addr_lookup2 */ na_bmi_addr_free, /* addr_free */ NULL, /* addr_set_remove */ na_bmi_addr_self, /* addr_self */ @@ -895,33 +890,13 @@ na_bmi_op_destroy(na_class_t NA_UNUSED *na_class, na_op_id_t op_id) /*---------------------------------------------------------------------------*/ static na_return_t -na_bmi_addr_lookup(na_class_t *na_class, na_context_t *context, - na_cb_t callback, void *arg, const char *name, na_op_id_t *op_id) +na_bmi_addr_lookup(na_class_t NA_UNUSED *na_class, const char *name, + na_addr_t *addr) { - struct na_bmi_op_id *na_bmi_op_id = NULL; struct na_bmi_addr *na_bmi_addr = NULL; na_return_t ret = NA_SUCCESS; int bmi_ret; - /* Allocate op_id if not provided */ - if (op_id && op_id != NA_OP_ID_IGNORE && *op_id != NA_OP_ID_NULL) { - na_bmi_op_id = (struct na_bmi_op_id *) *op_id; - hg_atomic_incr32(&na_bmi_op_id->ref_count); - } else { - na_bmi_op_id = (struct na_bmi_op_id *) na_bmi_op_create(na_class); - if (!na_bmi_op_id) { - NA_LOG_ERROR("Could not allocate NA BMI operation ID"); - ret = NA_NOMEM_ERROR; - goto done; - } - } - na_bmi_op_id->context = context; - na_bmi_op_id->type = NA_CB_LOOKUP; - na_bmi_op_id->callback = callback; - na_bmi_op_id->arg = arg; - hg_atomic_set32(&na_bmi_op_id->completed, 0); - na_bmi_op_id->cancel = 0; - /* Allocate addr */ na_bmi_addr = (struct na_bmi_addr *) malloc(sizeof(struct na_bmi_addr)); if (!na_bmi_addr) { @@ -933,11 +908,6 @@ na_bmi_addr_lookup(na_class_t *na_class, na_context_t *context, na_bmi_addr->unexpected = NA_FALSE; na_bmi_addr->self = NA_FALSE; hg_atomic_set32(&na_bmi_addr->ref_count, 1); - na_bmi_op_id->info.lookup.addr = (na_addr_t) na_bmi_addr; - - /* Assign op_id */ - if (op_id && op_id != NA_OP_ID_IGNORE && *op_id == NA_OP_ID_NULL) - *op_id = na_bmi_op_id; /* Perform an address lookup */ bmi_ret = BMI_addr_lookup(&na_bmi_addr->bmi_addr, name); @@ -947,17 +917,11 @@ na_bmi_addr_lookup(na_class_t *na_class, na_context_t *context, goto done; } - /* TODO we always complete here for now as lookup is blocking */ - ret = na_bmi_complete(na_bmi_op_id); - if (ret != NA_SUCCESS) { - NA_LOG_ERROR("Could not complete operation"); - goto done; - } + *addr = (na_addr_t) na_bmi_addr; done: if (ret != NA_SUCCESS) { free(na_bmi_addr); - na_bmi_op_destroy(na_class, (na_op_id_t) na_bmi_op_id); } return ret; } @@ -2071,9 +2035,6 @@ na_bmi_progress_expected(na_class_t NA_UNUSED *na_class, na_context_t *context, } switch (na_bmi_op_id->type) { - case NA_CB_LOOKUP: - NA_LOG_ERROR("Should not complete lookup here"); - break; case NA_CB_RECV_UNEXPECTED: NA_LOG_ERROR("Should not complete unexpected recv here"); break; @@ -2344,9 +2305,6 @@ na_bmi_complete(struct na_bmi_op_id *na_bmi_op_id) callback_info->type = na_bmi_op_id->type; switch (na_bmi_op_id->type) { - case NA_CB_LOOKUP: - callback_info->info.lookup.addr = na_bmi_op_id->info.lookup.addr; - break; case NA_CB_SEND_UNEXPECTED: break; case NA_CB_RECV_UNEXPECTED: @@ -2469,9 +2427,6 @@ na_bmi_cancel(na_class_t *na_class, na_context_t *context, na_op_id_t op_id) goto done; switch (na_bmi_op_id->type) { - case NA_CB_LOOKUP: - /* Nothing for now */ - break; case NA_CB_SEND_UNEXPECTED: bmi_ret = BMI_cancel(na_bmi_op_id->info.send_unexpected.op_id, *bmi_context); diff --git a/src/na/na_cci.c b/src/na/na_cci.c index fabe5b1a..3d5ebb1e 100644 --- a/src/na/na_cci.c +++ b/src/na/na_cci.c @@ -60,7 +60,6 @@ struct na_cci_addr { HG_QUEUE_HEAD(na_cci_op_id) rxs; /* Posted recvs */ HG_QUEUE_HEAD(na_cci_info_recv_expected) early; /* Expected recvs not yet posted */ char *uri; /* Peer's URI */ - na_cci_op_id_t *na_cci_op_id; /* For addr_lookup() */ hg_atomic_int32_t refcnt; /* Reference counter */ na_bool_t unexpected; /* Address generated from unexpected recv */ na_bool_t self; /* Boolean for self */ @@ -79,10 +78,6 @@ typedef enum na_cci_rma_op { NA_CCI_RMA_GET /* Request a get operation */ } na_cci_rma_op_t; -struct na_cci_info_lookup { - na_addr_t addr; -}; - struct na_cci_info_send_unexpected { cci_op_id_t op_id; /* CCI operation ID */ }; @@ -141,7 +136,6 @@ struct na_cci_op_id { hg_atomic_int32_t completed; /* Operation completed */ hg_atomic_int32_t canceled; /* Operation canceled */ union { - struct na_cci_info_lookup lookup; struct na_cci_info_send_unexpected send_unexpected; struct na_cci_info_recv_unexpected recv_unexpected; struct na_cci_info_send_expected send_expected; @@ -223,8 +217,7 @@ na_cci_op_destroy(na_class_t *na_class, na_op_id_t op_id); /* addr_lookup */ static na_return_t -na_cci_addr_lookup(na_class_t * na_class, na_context_t * context, - na_cb_t callback, void *arg, const char *name, na_op_id_t * op_id); +na_cci_addr_lookup(na_class_t * na_class, const char *name, na_addr_t * addr); /* addr_self */ static na_return_t @@ -379,7 +372,6 @@ const struct na_class_ops NA_PLUGIN_OPS(cci) = { na_cci_op_create, /* op_create */ na_cci_op_destroy, /* op_destroy */ na_cci_addr_lookup, /* addr_lookup */ - NULL, /* addr_lookup2 */ na_cci_addr_free, /* addr_free */ NULL, /* addr_set_remove */ na_cci_addr_self, /* addr_self */ @@ -786,35 +778,14 @@ na_cci_op_destroy(na_class_t NA_UNUSED *na_class, na_op_id_t op_id) /*---------------------------------------------------------------------------*/ static na_return_t -na_cci_addr_lookup(na_class_t * na_class, na_context_t * context, - na_cb_t callback, void *arg, const char *name, na_op_id_t * op_id) +na_cci_addr_lookup(na_class_t * na_class, const char *name, na_addr_t * addr) { cci_endpoint_t *e = NA_CCI_CLASS(na_class)->endpoint; char *uri = NA_CCI_CLASS(na_class)->uri; - struct na_cci_op_id *na_cci_op_id = NULL; na_cci_addr_t *na_cci_addr = NULL; na_return_t ret = NA_SUCCESS; int rc; - /* Allocate op_id if not provided */ - if (op_id && op_id != NA_OP_ID_IGNORE && *op_id != NA_OP_ID_NULL) { - na_cci_op_id = (na_cci_op_id_t *) *op_id; - hg_atomic_incr32(&na_cci_op_id->refcnt); - } else { - na_cci_op_id = (na_cci_op_id_t *) na_cci_op_create(na_class); - if (!na_cci_op_id) { - NA_LOG_ERROR("Could not create NA CCI operation ID"); - ret = NA_NOMEM_ERROR; - goto out; - } - } - na_cci_op_id->context = context; - na_cci_op_id->type = NA_CB_LOOKUP; - na_cci_op_id->callback = callback; - na_cci_op_id->arg = arg; - hg_atomic_set32(&na_cci_op_id->completed, 0); - hg_atomic_set32(&na_cci_op_id->canceled, 0); - /* Allocate addr */ na_cci_addr = (na_cci_addr_t *) malloc(sizeof(*na_cci_addr)); if (!na_cci_addr) { @@ -832,13 +803,8 @@ na_cci_addr_lookup(na_class_t * na_class, na_context_t * context, hg_atomic_set32(&na_cci_addr->refcnt, 1); na_cci_addr->unexpected = NA_FALSE; na_cci_addr->self = NA_FALSE; - na_cci_addr->na_cci_op_id = na_cci_op_id; - na_cci_op_id->info.lookup.addr = (na_addr_t) na_cci_addr; - - /* Assign op_id */ - if (op_id && op_id != NA_OP_ID_IGNORE && *op_id == NA_OP_ID_NULL) - *op_id = (na_op_id_t) na_cci_op_id; + /* TODO we would need to ensure that connect completes before using the addr */ rc = cci_connect(e, name, uri, (uint32_t) strlen(uri) + 1, CCI_CONN_ATTR_RO, na_cci_addr, 0, NULL); if (rc) { @@ -851,10 +817,11 @@ na_cci_addr_lookup(na_class_t * na_class, na_context_t * context, goto out; } + *addr = (na_addr_t) na_cci_addr; + out: if (ret != NA_SUCCESS) { free(na_cci_addr); - na_cci_op_destroy(na_class, na_cci_op_id); } return ret; } @@ -877,7 +844,6 @@ na_cci_addr_self(na_class_t * na_class, na_addr_t * addr) na_cci_addr->uri = strdup(NA_CCI_CLASS(na_class)->uri); na_cci_addr->unexpected = NA_FALSE; na_cci_addr->self = NA_TRUE; - na_cci_addr->na_cci_op_id = NULL; hg_atomic_set32(&na_cci_addr->refcnt, 1); *addr = (na_addr_t) na_cci_addr; @@ -1982,31 +1948,14 @@ handle_connect(na_class_t NA_UNUSED *class, na_context_t NA_UNUSED *context, cci_endpoint_t *e, cci_event_t *event) { na_cci_addr_t *na_cci_addr = event->connect.context; - na_cci_op_id_t *na_cci_op_id = na_cci_addr->na_cci_op_id; - na_return_t ret = NA_SUCCESS; - - if (!na_cci_addr->na_cci_op_id) { - /* User canceled lookup */ - addr_decref(na_cci_addr); - op_id_decref(na_cci_op_id); - goto out; - } - - na_cci_addr->na_cci_op_id = NULL; if (event->connect.status != CCI_SUCCESS) { NA_LOG_ERROR("connect to %s failed with %s", na_cci_addr->uri, cci_strerror(e, event->connect.status)); - ret = NA_PROTOCOL_ERROR; } else { na_cci_addr->cci_addr = event->connect.connection; } - ret = na_cci_complete(na_cci_addr, na_cci_op_id, ret); - if (ret != NA_SUCCESS) - NA_LOG_ERROR("Could not complete operation"); - -out: return; } @@ -2113,10 +2062,6 @@ na_cci_complete(na_cci_addr_t *na_cci_addr, na_cci_op_id_t *na_cci_op_id, callback_info->type = na_cci_op_id->type; switch (na_cci_op_id->type) { - case NA_CB_LOOKUP: - addr_addref(na_cci_op_id->info.lookup.addr); - callback_info->info.lookup.addr = na_cci_op_id->info.lookup.addr; - break; case NA_CB_RECV_UNEXPECTED: { /* Fill callback info */ callback_info->info.recv_unexpected.actual_buf_size = @@ -2193,13 +2138,6 @@ na_cci_cancel(na_class_t * na_class, na_context_t NA_UNUSED * context, op_id_addref(na_cci_op_id); /* will be decremented in handle_*() */ switch (na_cci_op_id->type) { - case NA_CB_LOOKUP: { - na_cci_addr = na_cci_op_id->info.lookup.addr; - - /* handle_connect() will need to cleanup the addr */ - na_cci_addr->na_cci_op_id = NULL; - } - break; case NA_CB_RECV_UNEXPECTED: { na_cci_op_id_t *tmp = NULL, *first = NULL; diff --git a/src/na/na_mpi.c b/src/na/na_mpi.c index bc6acda7..7c9a56b9 100644 --- a/src/na/na_mpi.c +++ b/src/na/na_mpi.c @@ -96,11 +96,6 @@ struct na_mpi_rma_info { na_tag_t tag; /* Tag used for the data transfer */ }; -/* na_mpi_info_lookup */ -struct na_mpi_info_lookup { - na_addr_t addr; -}; - /* na_mpi_info_send_unexpected */ struct na_mpi_info_send_unexpected { MPI_Request data_request; @@ -152,7 +147,6 @@ struct na_mpi_op_id { hg_atomic_int32_t completed; /* Operation completed */ na_bool_t canceled; /* Operation canceled */ union { - struct na_mpi_info_lookup lookup; struct na_mpi_info_send_unexpected send_unexpected; struct na_mpi_info_recv_unexpected recv_unexpected; struct na_mpi_info_send_expected send_expected; @@ -287,11 +281,8 @@ na_mpi_op_destroy( static na_return_t na_mpi_addr_lookup( na_class_t *na_class, - na_context_t *context, - na_cb_t callback, - void *arg, const char *name, - na_op_id_t *op_id + na_addr_t *addr ); /* addr_self */ @@ -571,7 +562,6 @@ const struct na_class_ops NA_PLUGIN_OPS(mpi) = { na_mpi_op_create, /* op_create */ na_mpi_op_destroy, /* op_destroy */ na_mpi_addr_lookup, /* addr_lookup */ - NULL, /* addr_lookup2 */ na_mpi_addr_free, /* addr_free */ NULL, /* addr_set_remove */ na_mpi_addr_self, /* addr_self */ @@ -637,7 +627,6 @@ na_mpi_accept_service(void *args) NA_LOG_ERROR("Could not accept connection"); } - hg_thread_exit(ret); return ret; } @@ -1285,23 +1274,12 @@ na_mpi_op_destroy(na_class_t NA_UNUSED *na_class, na_op_id_t op_id) /*---------------------------------------------------------------------------*/ static na_return_t -na_mpi_addr_lookup(na_class_t *na_class, na_context_t *context, - na_cb_t callback, void *arg, const char *name, na_op_id_t *op_id) +na_mpi_addr_lookup(na_class_t *na_class, const char *name, na_addr_t *addr) { - struct na_mpi_op_id *na_mpi_op_id = NULL; struct na_mpi_addr *na_mpi_addr = NULL; na_return_t ret = NA_SUCCESS; int mpi_ret; - na_mpi_op_id = (struct na_mpi_op_id *) *op_id; - hg_atomic_incr32(&na_mpi_op_id->ref_count); - na_mpi_op_id->context = context; - na_mpi_op_id->type = NA_CB_LOOKUP; - na_mpi_op_id->callback = callback; - na_mpi_op_id->arg = arg; - hg_atomic_set32(&na_mpi_op_id->completed, 0); - na_mpi_op_id->canceled = NA_FALSE; - /* Allocate addr */ na_mpi_addr = (struct na_mpi_addr *) malloc(sizeof(struct na_mpi_addr)); if (!na_mpi_addr) { @@ -1315,7 +1293,7 @@ na_mpi_addr_lookup(na_class_t *na_class, na_context_t *context, na_mpi_addr->unexpected = NA_FALSE; na_mpi_addr->self = NA_FALSE; na_mpi_addr->dynamic = NA_FALSE; - na_mpi_op_id->info.lookup.addr = (na_addr_t) na_mpi_addr; + memset(na_mpi_addr->port_name, '\0', MPI_MAX_PORT_NAME); /* get port_name and remote server rank */ na_mpi_get_port_info(name, na_mpi_addr->port_name, &na_mpi_addr->rank); @@ -1377,18 +1355,11 @@ na_mpi_addr_lookup(na_class_t *na_class, na_context_t *context, na_mpi_addr, entry); hg_thread_mutex_unlock(&NA_MPI_CLASS(na_class)->remote_list_mutex); - /* TODO MPI calls are blocking and so is na_mpi_addr_lookup, - * i.e. we always complete here for now */ - ret = na_mpi_complete(na_mpi_op_id); - if (ret != NA_SUCCESS) { - NA_LOG_ERROR("Could not complete operation"); - goto done; - } + *addr = (na_addr_t) na_mpi_addr; done: if (ret != NA_SUCCESS) { free(na_mpi_addr); - na_mpi_op_destroy(na_class, (na_op_id_t) na_mpi_op_id); } return ret; @@ -2344,9 +2315,6 @@ na_mpi_progress_expected(na_class_t *na_class, na_context_t NA_UNUSED *context, } switch (na_mpi_op_id->type) { - case NA_CB_LOOKUP: - NA_LOG_ERROR("Should not complete lookup here"); - break; case NA_CB_RECV_UNEXPECTED: NA_LOG_ERROR("Should not complete unexpected recv here"); break; @@ -2465,9 +2433,6 @@ na_mpi_complete(struct na_mpi_op_id *na_mpi_op_id) callback_info->type = na_mpi_op_id->type; switch (na_mpi_op_id->type) { - case NA_CB_LOOKUP: - callback_info->info.lookup.addr = na_mpi_op_id->info.lookup.addr; - break; case NA_CB_SEND_UNEXPECTED: break; case NA_CB_RECV_UNEXPECTED: @@ -2598,9 +2563,6 @@ na_mpi_cancel(na_class_t *na_class, na_context_t NA_UNUSED *context, goto done; switch (na_mpi_op_id->type) { - case NA_CB_LOOKUP: - /* Nothing for now */ - break; case NA_CB_SEND_UNEXPECTED: mpi_ret = MPI_Cancel(&na_mpi_op_id->info.send_unexpected.data_request); if (mpi_ret != MPI_SUCCESS) { diff --git a/src/na/na_ofi.c b/src/na/na_ofi.c index 16534f59..c6d7711d 100644 --- a/src/na/na_ofi.c +++ b/src/na/na_ofi.c @@ -943,8 +943,7 @@ const struct na_class_ops NA_PLUGIN_OPS(ofi) = { na_ofi_context_destroy, /* context_destroy */ na_ofi_op_create, /* op_create */ na_ofi_op_destroy, /* op_destroy */ - NULL, /* addr_lookup */ - na_ofi_addr_lookup, /* addr_lookup2 */ + na_ofi_addr_lookup, /* addr_lookup */ na_ofi_addr_free, /* addr_free */ na_ofi_addr_set_remove, /* addr_set_remove */ na_ofi_addr_self, /* addr_self */ @@ -1277,12 +1276,13 @@ na_ofi_addr_ht_lookup(struct na_ofi_domain *domain, na_uint32_t addr_format, /* Lookup key */ hg_thread_rwlock_rdlock(&domain->rwlock); ht_value = hg_hash_table_lookup(domain->addr_ht, ht_key); - hg_thread_rwlock_release_rdlock(&domain->rwlock); if (ht_value != HG_HASH_TABLE_NULL) { /* Found */ *fi_addr = *(fi_addr_t *) ht_value; + hg_thread_rwlock_release_rdlock(&domain->rwlock); goto out; } + hg_thread_rwlock_release_rdlock(&domain->rwlock); /* Insert addr into AV if key not found */ na_ofi_domain_lock(domain); @@ -1344,10 +1344,16 @@ static na_return_t na_ofi_addr_ht_remove(struct na_ofi_domain *domain, fi_addr_t *fi_addr, na_uint64_t *addr_key) { + hg_hash_table_value_t ht_value = NULL; na_return_t ret = NA_SUCCESS; int rc; hg_thread_rwlock_wrlock(&domain->rwlock); + ht_value = hg_hash_table_lookup(domain->addr_ht, + (hg_hash_table_key_t) addr_key); + if (ht_value == HG_HASH_TABLE_NULL) + goto unlock; + rc = hg_hash_table_remove(domain->addr_ht, (hg_hash_table_key_t) addr_key); NA_CHECK_ERROR(rc != 1, unlock, ret, NA_NOENTRY, "hg_hash_table_remove() failed"); @@ -2825,7 +2831,6 @@ na_ofi_cq_process_retries(na_context_t *context) rc = fi_readmsg(ctx->fi_tx, &na_ofi_op_id->info.rma.fi_rma, NA_OFI_GET_COMPLETION); break; - case NA_CB_LOOKUP: default: NA_GOTO_ERROR(error, ret, NA_INVALID_ARG, "Operation type %d not supported", @@ -2926,7 +2931,6 @@ na_ofi_complete(struct na_ofi_op_id *na_ofi_op_id) case NA_CB_PUT: case NA_CB_GET: break; - case NA_CB_LOOKUP: default: NA_GOTO_ERROR(out, ret, NA_INVALID_ARG, "Operation type %d not supported", callback_info->type); @@ -4636,7 +4640,6 @@ na_ofi_cancel(na_class_t *na_class, na_context_t *context, case NA_CB_GET: fi_ep = NA_OFI_CONTEXT(context)->fi_tx; break; - case NA_CB_LOOKUP: default: NA_GOTO_ERROR(out, ret, NA_INVALID_ARG, "Operation type %d not supported", diff --git a/src/na/na_types.h b/src/na/na_types.h index 7032bd03..9245e0fd 100644 --- a/src/na/na_types.h +++ b/src/na/na_types.h @@ -74,7 +74,6 @@ typedef enum na_return { NA_RETURN_VALUES } na_return_t; /* Callback operation type */ typedef enum na_cb_type { - NA_CB_LOOKUP, /*!< lookup callback */ NA_CB_SEND_UNEXPECTED, /*!< unexpected send callback */ NA_CB_RECV_UNEXPECTED, /*!< unexpected recv callback */ NA_CB_SEND_EXPECTED, /*!< expected send callback */ @@ -84,10 +83,6 @@ typedef enum na_cb_type { } na_cb_type_t; /* Callback info structs */ -struct na_cb_info_lookup { - na_addr_t addr; -}; - struct na_cb_info_recv_unexpected { na_size_t actual_buf_size; na_addr_t source; @@ -97,7 +92,6 @@ struct na_cb_info_recv_unexpected { /* Callback info struct */ struct na_cb_info { union { /* Union of callback info structures */ - struct na_cb_info_lookup lookup; struct na_cb_info_recv_unexpected recv_unexpected; } info; void *arg; /* User data */ @@ -121,9 +115,13 @@ typedef int (*na_cb_t)(const struct na_cb_info *callback_info); /* Max timeout */ #define NA_MAX_IDLE_TIME (3600*1000) -/* Tag upper bound - * \remark This is not the user tag limit but only the limit imposed by the type */ -#define NA_TAG_UB UINT_MAX +/* Context ID max value + * \remark This is not the user limit but only the limit imposed by the type */ +#define NA_CONTEXT_ID_MAX UINT8_MAX + +/* Tag max value + * \remark This is not the user limit but only the limit imposed by the type */ +#define NA_TAG_MAX UINT_MAX /* The memory attributes associated with the memory handle * can be defined as read only, write only or read/write */ From d1185bab34a54665fe69348eb02eca18da645295 Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Thu, 14 May 2020 18:37:58 -0500 Subject: [PATCH 09/30] HG: Add HG/Core_addr_lookup2() for immediate lookups HG_Core_addr_lookup1() now also completes immediately Switch to Addr_lookup1() in HL library and update tests Map HG_Addr_lookup() to HG_Addr_lookup1() by default --- Testing/test_rpc.c | 8 +- src/mercury.c | 30 ++++-- src/mercury.h | 28 +++++- src/mercury_core.c | 233 ++++++++++++++++----------------------------- src/mercury_core.h | 27 +++++- src/mercury_hl.c | 2 +- 6 files changed, 159 insertions(+), 169 deletions(-) diff --git a/Testing/test_rpc.c b/Testing/test_rpc.c index 77c8fa38..0d8418a2 100644 --- a/Testing/test_rpc.c +++ b/Testing/test_rpc.c @@ -299,9 +299,9 @@ hg_test_rpc_lookup(hg_context_t *context, hg_request_class_t *request_class, lookup_args.request = request; /* Forward call to remote addr and get a new request */ - ret = HG_Addr_lookup(context, hg_test_rpc_lookup_cb, + ret = HG_Addr_lookup1(context, hg_test_rpc_lookup_cb, &lookup_args, target_name, HG_OP_ID_IGNORE); - HG_TEST_CHECK_HG_ERROR(done, ret, "HG_Addr_lookup() failed (%s)", + HG_TEST_CHECK_HG_ERROR(done, ret, "HG_Addr_lookup1() failed (%s)", HG_Error_to_string(ret)); /* Wait for request to be marked completed */ @@ -709,11 +709,11 @@ main(int argc, char *argv[]) lookup_args.request = request; /* Forward call to remote addr and get a new request */ - hg_ret = HG_Addr_lookup(hg_test_info.context, hg_test_rpc_lookup_cb, + hg_ret = HG_Addr_lookup1(hg_test_info.context, hg_test_rpc_lookup_cb, &lookup_args, hg_test_info.na_test_info.target_name, HG_OP_ID_IGNORE); HG_TEST_CHECK_ERROR(hg_ret != HG_SUCCESS, done, ret, EXIT_FAILURE, - "HG_Addr_lookup() failed (%s)", HG_Error_to_string(hg_ret)); + "HG_Addr_lookup1() failed (%s)", HG_Error_to_string(hg_ret)); /* Wait for request to be marked completed */ hg_request_wait(request, HG_MAX_IDLE_TIME, &flag); diff --git a/src/mercury.c b/src/mercury.c index 4101ffe0..458f8b39 100644 --- a/src/mercury.c +++ b/src/mercury.c @@ -75,7 +75,6 @@ struct hg_private_handle { /* HG op id */ struct hg_op_info_lookup { struct hg_addr *hg_addr; /* Address */ - hg_core_op_id_t core_op_id; /* Operation ID for lookup */ }; struct hg_op_id { @@ -1442,7 +1441,7 @@ HG_Registered_disabled_response(hg_class_t *hg_class, hg_id_t id, /*---------------------------------------------------------------------------*/ hg_return_t -HG_Addr_lookup(hg_context_t *context, hg_cb_t callback, void *arg, +HG_Addr_lookup1(hg_context_t *context, hg_cb_t callback, void *arg, const char *name, hg_op_id_t *op_id) { struct hg_op_id *hg_op_id = NULL; @@ -1450,6 +1449,7 @@ HG_Addr_lookup(hg_context_t *context, hg_cb_t callback, void *arg, HG_CHECK_ERROR(context == NULL, error, ret, HG_INVALID_ARG, "NULL HG context"); + (void) op_id; /* Allocate op_id */ hg_op_id = (struct hg_op_id *) malloc(sizeof(struct hg_op_id)); @@ -1462,15 +1462,11 @@ HG_Addr_lookup(hg_context_t *context, hg_cb_t callback, void *arg, hg_op_id->arg = arg; hg_op_id->info.lookup.hg_addr = HG_ADDR_NULL; - ret = HG_Core_addr_lookup(context->core_context, hg_core_addr_lookup_cb, - hg_op_id, name, &hg_op_id->info.lookup.core_op_id); + ret = HG_Core_addr_lookup1(context->core_context, hg_core_addr_lookup_cb, + hg_op_id, name, HG_CORE_OP_ID_IGNORE); HG_CHECK_HG_ERROR(error, ret, "Could not lookup %s (%s)", name, HG_Error_to_string(ret)); - /* Assign op_id */ - if (op_id && op_id != HG_OP_ID_IGNORE) - *op_id = (hg_op_id_t) hg_op_id; - return ret; error: @@ -1479,6 +1475,24 @@ HG_Addr_lookup(hg_context_t *context, hg_cb_t callback, void *arg, return ret; } +/*---------------------------------------------------------------------------*/ +hg_return_t +HG_Addr_lookup2(hg_class_t *hg_class, const char *name, hg_addr_t *addr) +{ + hg_return_t ret = HG_SUCCESS; + + HG_CHECK_ERROR(hg_class == NULL, done, ret, HG_INVALID_ARG, + "NULL HG class"); + + ret = HG_Core_addr_lookup2(hg_class->core_class, name, + (hg_core_addr_t *) addr); + HG_CHECK_HG_ERROR(done, ret, "Could not lookup %s (%s)", name, + HG_Error_to_string(ret)); + +done: + return ret; +} + /*---------------------------------------------------------------------------*/ hg_return_t HG_Addr_free(hg_class_t *hg_class, hg_addr_t addr) diff --git a/src/mercury.h b/src/mercury.h index d8d385d9..326dd3ec 100644 --- a/src/mercury.h +++ b/src/mercury.h @@ -574,12 +574,12 @@ HG_Registered_disabled_response( * \param callback [IN] pointer to function callback * \param arg [IN] pointer to data passed to callback * \param name [IN] lookup name - * \param op_id [OUT] pointer to returned operation ID + * \param op_id [OUT] pointer to returned operation ID (unused) * * \return HG_SUCCESS or corresponding HG error code */ HG_PUBLIC hg_return_t -HG_Addr_lookup( +HG_Addr_lookup1( hg_context_t *context, hg_cb_t callback, void *arg, @@ -587,6 +587,30 @@ HG_Addr_lookup( hg_op_id_t *op_id ); +/* This will map to HG_Addr_lookup2() in the future */ +#ifndef HG_Addr_lookup +#define HG_Addr_lookup HG_Addr_lookup1 +#endif + +/** + * Lookup an addr from a peer address/name. Addresses need to be + * freed by calling HG_Addr_free(). + * + * \remark This is the immediate version of HG_Addr_lookup1(). + * + * \param hg_class [IN/OUT] pointer to HG class + * \param name [IN] lookup name + * \param addr [OUT] pointer to abstract address + * + * \return HG_SUCCESS or corresponding HG error code + */ +HG_PUBLIC hg_return_t +HG_Addr_lookup2( + hg_class_t *hg_class, + const char *name, + hg_addr_t *addr + ); + /** * Free the addr. * diff --git a/src/mercury_core.c b/src/mercury_core.c index cd4587ae..15f45b8a 100644 --- a/src/mercury_core.c +++ b/src/mercury_core.c @@ -212,7 +212,6 @@ struct hg_core_private_handle { /* HG op id */ struct hg_core_op_info_lookup { struct hg_core_private_addr *hg_core_addr; /* Address */ - na_op_id_t na_lookup_op_id; /* Operation ID for lookup */ }; struct hg_core_op_id { @@ -341,27 +340,9 @@ hg_core_addr_create( */ static hg_return_t hg_core_addr_lookup( - struct hg_core_private_context *context, - hg_core_cb_t callback, - void *arg, + struct hg_core_private_class *hg_core_class, const char *name, - hg_core_op_id_t *op_id - ); - -/** - * Lookup callback. - */ -static int -hg_core_addr_lookup_cb( - const struct na_cb_info *callback_info - ); - -/** - * Complete addr lookup. - */ -static hg_return_t -hg_core_addr_lookup_complete( - struct hg_core_op_id *hg_core_op_id + struct hg_core_private_addr **addr ); /** @@ -1228,38 +1209,22 @@ hg_core_addr_create(struct hg_core_private_class *hg_core_class, /*---------------------------------------------------------------------------*/ static hg_return_t -hg_core_addr_lookup(struct hg_core_private_context *context, - hg_core_cb_t callback, void *arg, const char *name, hg_core_op_id_t *op_id) +hg_core_addr_lookup(struct hg_core_private_class *hg_core_class, + const char *name, struct hg_core_private_addr **addr) { - na_class_t *na_class = context->core_context.core_class->na_class; - na_context_t *na_context = context->core_context.na_context; - struct hg_core_op_id *hg_core_op_id = NULL; + na_class_t *na_class = hg_core_class->core_class.na_class; struct hg_core_private_addr *hg_core_addr = NULL; - na_addr_t na_addr = NA_ADDR_NULL; na_return_t na_ret; #ifdef HG_HAS_SM_ROUTING char lookup_name[HG_CORE_ADDR_MAX_SIZE] = {'\0'}; #endif const char *name_str = name; - hg_return_t ret = HG_SUCCESS, progress_ret; - - /* Allocate op_id */ - hg_core_op_id = (struct hg_core_op_id *) malloc( - sizeof(struct hg_core_op_id)); - HG_CHECK_ERROR(hg_core_op_id == NULL, error, ret, HG_NOMEM, - "Could not allocate HG operation ID"); - - hg_core_op_id->context = context; - hg_core_op_id->type = HG_CB_LOOKUP; - hg_core_op_id->callback = callback; - hg_core_op_id->arg = arg; - hg_core_op_id->info.lookup.hg_core_addr = NULL; + hg_return_t ret = HG_SUCCESS; /* Allocate addr */ - hg_core_addr = hg_core_addr_create(HG_CORE_CONTEXT_CLASS(context), NULL); + hg_core_addr = hg_core_addr_create(hg_core_class, NULL); HG_CHECK_ERROR(hg_core_addr == NULL, error, ret, HG_NOMEM, "Could not create HG addr"); - hg_core_op_id->info.lookup.hg_core_addr = hg_core_addr; #ifdef HG_HAS_SM_ROUTING /* Parse name string */ @@ -1284,12 +1249,10 @@ hg_core_addr_lookup(struct hg_core_private_context *context, local_name = lookup_names; /* Compare UUIDs, if they match it's local address */ - if (context->core_context.na_sm_context - && uuid_compare(hg_core_addr->na_sm_uuid, - HG_CORE_CONTEXT_CLASS(context)->na_sm_uuid) == 0) { + if (hg_core_class->core_class.na_sm_class && uuid_compare( + hg_core_addr->na_sm_uuid, hg_core_class->na_sm_uuid) == 0) { name_str = local_name; - na_class = context->core_context.core_class->na_sm_class; - na_context = context->core_context.na_sm_context; + na_class = hg_core_class->core_class.na_sm_class; } else { /* Remote lookup */ name_str = remote_name; @@ -1299,95 +1262,20 @@ hg_core_addr_lookup(struct hg_core_private_context *context, /* Assign corresponding NA class */ hg_core_addr->core_addr.na_class = na_class; - /* Try to use immediate lookup */ - na_ret = NA_Addr_lookup2(na_class, name_str, &na_addr); + /* Lookup adress */ + na_ret = NA_Addr_lookup(na_class, name_str, + &hg_core_addr->core_addr.na_addr); HG_CHECK_ERROR(na_ret != NA_SUCCESS, error, ret, (hg_return_t) na_ret, - "Could not start lookup for address %s (%s)", name_str, + "Could not lookup address %s (%s)", name_str, NA_Error_to_string(na_ret)); - if (na_addr != NA_ADDR_NULL) { - struct na_cb_info callback_info; - callback_info.arg = hg_core_op_id; - callback_info.ret = NA_SUCCESS; - callback_info.type = NA_CB_LOOKUP; - callback_info.info.lookup.addr = na_addr; - hg_core_op_id->info.lookup.na_lookup_op_id = NA_OP_ID_NULL; - - hg_core_addr_lookup_cb(&callback_info); - } else { - /* Create operation ID */ - hg_core_op_id->info.lookup.na_lookup_op_id = NA_Op_create(na_class); - - na_ret = NA_Addr_lookup(na_class, na_context, hg_core_addr_lookup_cb, - hg_core_op_id, name_str, &hg_core_op_id->info.lookup.na_lookup_op_id); - HG_CHECK_ERROR(na_ret != NA_SUCCESS, error, ret, (hg_return_t) na_ret, - "Could not start lookup for address %s (%s)", name_str, - NA_Error_to_string(na_ret)); - } - - /* TODO to avoid blocking after lookup make progress on the HG layer with - * timeout of 0 */ - progress_ret = context->progress(context, 0); - HG_CHECK_ERROR(progress_ret != HG_SUCCESS && progress_ret != HG_TIMEOUT, - error, ret, progress_ret, "Could not make progress"); - - /* Assign op_id */ - if (op_id && op_id != HG_CORE_OP_ID_IGNORE) - *op_id = (hg_core_op_id_t) hg_core_op_id; + *addr = hg_core_addr; return ret; error: - free(hg_core_op_id); - hg_core_addr_free(HG_CORE_CONTEXT_CLASS(context), hg_core_addr); - - return ret; -} - -/*---------------------------------------------------------------------------*/ -static int -hg_core_addr_lookup_cb(const struct na_cb_info *callback_info) -{ - struct hg_core_op_id *hg_core_op_id = - (struct hg_core_op_id *) callback_info->arg; - na_return_t na_ret = callback_info->ret; - hg_return_t hg_ret; - int ret = 0; - - HG_CHECK_ERROR_NORET(na_ret != NA_SUCCESS, done, "(%s)", - NA_Error_to_string(na_ret)); - - /* Assign addr */ - hg_core_op_id->info.lookup.hg_core_addr->core_addr.na_addr = - callback_info->info.lookup.addr; - - /* Mark as completed */ - hg_ret = hg_core_addr_lookup_complete(hg_core_op_id); - HG_CHECK_HG_ERROR(done, hg_ret, "Could not complete operation"); - - ret++; - -done: - return ret; -} - -/*---------------------------------------------------------------------------*/ -static hg_return_t -hg_core_addr_lookup_complete(struct hg_core_op_id *hg_core_op_id) -{ - hg_core_context_t *context = &hg_core_op_id->context->core_context; - struct hg_completion_entry *hg_completion_entry = - &hg_core_op_id->hg_completion_entry; - hg_return_t ret = HG_SUCCESS; - - hg_completion_entry->op_type = HG_ADDR; - hg_completion_entry->op_id.hg_core_op_id = hg_core_op_id; - - ret = hg_core_completion_add(context, hg_completion_entry, HG_FALSE); - HG_CHECK_HG_ERROR(done, ret, - "Could not add HG completion entry to completion queue"); + hg_core_addr_free(hg_core_class, hg_core_addr); -done: return ret; } @@ -3258,21 +3146,12 @@ hg_core_trigger_lookup_entry(struct hg_core_op_id *hg_core_op_id) { hg_return_t ret = HG_SUCCESS; - /* Free op */ - if (hg_core_op_id->info.lookup.na_lookup_op_id != NA_OP_ID_NULL) { - na_return_t na_ret = NA_Op_destroy( - hg_core_op_id->info.lookup.hg_core_addr->core_addr.na_class, - hg_core_op_id->info.lookup.na_lookup_op_id); - HG_CHECK_ERROR(na_ret != NA_SUCCESS, done, ret, (hg_return_t) na_ret, - "Could not destroy addr op ID (%s)", NA_Error_to_string(na_ret)); - } - /* Execute callback */ if (hg_core_op_id->callback) { struct hg_core_cb_info hg_core_cb_info; hg_core_cb_info.arg = hg_core_op_id->arg; - hg_core_cb_info.ret = HG_SUCCESS; /* TODO report failure */ + hg_core_cb_info.ret = HG_SUCCESS; hg_core_cb_info.type = HG_CB_LOOKUP; hg_core_cb_info.info.lookup.addr = (hg_core_addr_t) hg_core_op_id->info.lookup.hg_core_addr; @@ -3280,8 +3159,8 @@ hg_core_trigger_lookup_entry(struct hg_core_op_id *hg_core_op_id) hg_core_op_id->callback(&hg_core_cb_info); } -done: free(hg_core_op_id); + return ret; } @@ -3972,9 +3851,32 @@ HG_Core_registered_data(hg_core_class_t *hg_core_class, hg_id_t id) /*---------------------------------------------------------------------------*/ hg_return_t -HG_Core_addr_lookup(hg_core_context_t *context, hg_core_cb_t callback, +HG_Core_addr_create(hg_core_class_t *hg_core_class, hg_core_addr_t *addr) +{ + hg_return_t ret = HG_SUCCESS; + + HG_CHECK_ERROR(hg_core_class == NULL, done, ret, HG_INVALID_ARG, + "NULL HG core class"); + HG_CHECK_ERROR(addr == NULL, done, ret, HG_INVALID_ARG, + "NULL pointer to address"); + + *addr = (hg_core_addr_t) hg_core_addr_create( + (struct hg_core_private_class *) hg_core_class, + hg_core_class->na_class); + HG_CHECK_ERROR(*addr == HG_CORE_ADDR_NULL, done, ret, HG_NOMEM, + "Could not create address"); + +done: + return ret; +} + +/*---------------------------------------------------------------------------*/ +hg_return_t +HG_Core_addr_lookup1(hg_core_context_t *context, hg_core_cb_t callback, void *arg, const char *name, hg_core_op_id_t *op_id) { + struct hg_core_op_id *hg_core_op_id = NULL; + struct hg_completion_entry *hg_completion_entry = NULL; hg_return_t ret = HG_SUCCESS; HG_CHECK_ERROR(context == NULL, done, ret, HG_INVALID_ARG, @@ -3983,31 +3885,64 @@ HG_Core_addr_lookup(hg_core_context_t *context, hg_core_cb_t callback, "NULL callback"); HG_CHECK_ERROR(name == NULL, done, ret, HG_INVALID_ARG, "NULL lookup"); + (void) op_id; - ret = hg_core_addr_lookup((struct hg_core_private_context *) context, - callback, arg, name, op_id); - HG_CHECK_HG_ERROR(done, ret, "Could not lookup address"); + /* Allocate op_id */ + hg_core_op_id = (struct hg_core_op_id *) malloc( + sizeof(struct hg_core_op_id)); + HG_CHECK_ERROR(hg_core_op_id == NULL, error, ret, HG_NOMEM, + "Could not allocate HG operation ID"); + + hg_core_op_id->context = (struct hg_core_private_context *) context; + hg_core_op_id->type = HG_CB_LOOKUP; + hg_core_op_id->callback = callback; + hg_core_op_id->arg = arg; + hg_core_op_id->info.lookup.hg_core_addr = NULL; + + ret = hg_core_addr_lookup( + (struct hg_core_private_class *) context->core_class, name, + &hg_core_op_id->info.lookup.hg_core_addr); + HG_CHECK_HG_ERROR(error, ret, "Could not lookup address"); + + /* Add callback to completion queue */ + hg_completion_entry = &hg_core_op_id->hg_completion_entry; + hg_completion_entry->op_type = HG_ADDR; + hg_completion_entry->op_id.hg_core_op_id = hg_core_op_id; + + ret = hg_core_completion_add(context, hg_completion_entry, HG_TRUE); + HG_CHECK_HG_ERROR(error, ret, + "Could not add HG completion entry to completion queue"); done: return ret; + +error: + if (hg_core_op_id) { + hg_core_addr_free((struct hg_core_private_class *) context->core_class, + hg_core_op_id->info.lookup.hg_core_addr); + free(hg_core_op_id); + } + + return ret; } /*---------------------------------------------------------------------------*/ hg_return_t -HG_Core_addr_create(hg_core_class_t *hg_core_class, hg_core_addr_t *addr) +HG_Core_addr_lookup2(hg_core_class_t *hg_core_class, const char *name, + hg_core_addr_t *addr) { hg_return_t ret = HG_SUCCESS; HG_CHECK_ERROR(hg_core_class == NULL, done, ret, HG_INVALID_ARG, "NULL HG core class"); + HG_CHECK_ERROR(name == NULL, done, ret, HG_INVALID_ARG, + "NULL lookup"); HG_CHECK_ERROR(addr == NULL, done, ret, HG_INVALID_ARG, "NULL pointer to address"); - *addr = (hg_core_addr_t) hg_core_addr_create( - (struct hg_core_private_class *) hg_core_class, - hg_core_class->na_class); - HG_CHECK_ERROR(*addr == HG_CORE_ADDR_NULL, done, ret, HG_NOMEM, - "Could not create address"); + ret = hg_core_addr_lookup((struct hg_core_private_class *) hg_core_class, + name, (struct hg_core_private_addr **) addr); + HG_CHECK_HG_ERROR(done, ret, "Could not lookup address"); done: return ret; diff --git a/src/mercury_core.h b/src/mercury_core.h index ce5974bd..4cfbe1d4 100644 --- a/src/mercury_core.h +++ b/src/mercury_core.h @@ -528,6 +528,20 @@ HG_Core_registered_data( hg_id_t id ); +/** + * Create a HG core address. + * + * \param hg_core_class [IN] pointer to HG core class + * \param new_addr [OUT] pointer to abstract address + * + * \return HG_SUCCESS or corresponding HG error code + */ +HG_PUBLIC hg_return_t +HG_Core_addr_create( + hg_core_class_t *hg_core_class, + hg_core_addr_t *addr + ); + /** * Lookup an addr from a peer address/name. Addresses need to be * freed by calling HG_Core_addr_free(). After completion, user callback is @@ -537,12 +551,12 @@ HG_Core_registered_data( * \param callback [IN] pointer to function callback * \param arg [IN] pointer to data passed to callback * \param name [IN] lookup name - * \param op_id [OUT] pointer to returned operation ID + * \param op_id [OUT] pointer to returned operation ID (unused) * * \return HG_SUCCESS or corresponding HG error code */ HG_PUBLIC hg_return_t -HG_Core_addr_lookup( +HG_Core_addr_lookup1( hg_core_context_t *context, hg_core_cb_t callback, void *arg, @@ -551,16 +565,19 @@ HG_Core_addr_lookup( ); /** - * Create a HG core address. + * Lookup an addr from a peer address/name. Addresses need to be + * freed by calling HG_Core_addr_free(). * * \param hg_core_class [IN] pointer to HG core class - * \param new_addr [OUT] pointer to abstract address + * \param name [IN] lookup name + * \param addr [OUT] pointer to abstract address * * \return HG_SUCCESS or corresponding HG error code */ HG_PUBLIC hg_return_t -HG_Core_addr_create( +HG_Core_addr_lookup2( hg_core_class_t *hg_core_class, + const char *name, hg_core_addr_t *addr ); diff --git a/src/mercury_hl.c b/src/mercury_hl.c index fdd62968..17050633 100644 --- a/src/mercury_hl.c +++ b/src/mercury_hl.c @@ -286,7 +286,7 @@ HG_Hl_addr_lookup_wait(hg_context_t *context, hg_request_class_t *request_class, request_args.request = request; /* Forward call to remote addr and get a new request */ - ret = HG_Addr_lookup(context, hg_hl_addr_lookup_cb, &request_args, name, + ret = HG_Addr_lookup1(context, hg_hl_addr_lookup_cb, &request_args, name, HG_OP_ID_IGNORE); HG_CHECK_HG_ERROR(done, ret, "Could not lookup address"); From e7b0ec76e23e2b549628d397658dab2a665e2ae4 Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Thu, 14 May 2020 18:40:21 -0500 Subject: [PATCH 10/30] HG Test: use HG_Addr_lookup2() for init lookups --- Testing/mercury_test.c | 43 ++---------------------------------------- 1 file changed, 2 insertions(+), 41 deletions(-) diff --git a/Testing/mercury_test.c b/Testing/mercury_test.c index 7cfadb24..b49e1ca3 100644 --- a/Testing/mercury_test.c +++ b/Testing/mercury_test.c @@ -27,11 +27,6 @@ /* Local Type and Struct Definition */ /************************************/ -struct hg_test_lookup_arg { - hg_addr_t *addr_ptr; - hg_request_t *request; -}; - /********************/ /* Local Prototypes */ /********************/ @@ -54,9 +49,6 @@ static hg_return_t hg_test_handle_create_cb(hg_handle_t handle, void *arg); #endif -static hg_return_t -hg_test_addr_lookup_cb(const struct hg_cb_info *callback_info); - static hg_return_t hg_test_finalize_rpc(struct hg_test_info *hg_test_info, hg_uint8_t target_id); @@ -205,20 +197,6 @@ hg_test_handle_create_cb(hg_handle_t handle, void *arg) } #endif -/*---------------------------------------------------------------------------*/ -static hg_return_t -hg_test_addr_lookup_cb(const struct hg_cb_info *callback_info) -{ - struct hg_test_lookup_arg *request_args = - (struct hg_test_lookup_arg *) callback_info->arg; - - *request_args->addr_ptr = callback_info->info.lookup.addr; - - hg_request_complete(request_args->request); - - return HG_SUCCESS; -} - /*---------------------------------------------------------------------------*/ static hg_return_t hg_test_finalize_rpc(struct hg_test_info *hg_test_info, hg_uint8_t target_id) @@ -559,9 +537,6 @@ HG_Test_init(int argc, char *argv[], struct hg_test_info *hg_test_info) HG_Error_to_string(ret)); } else { char test_addr_name[NA_TEST_MAX_ADDR_NAME] = { '\0' }; - hg_request_t *request = NULL; - unsigned int flag = 0; - struct hg_test_lookup_arg lookup_args; #ifdef HG_TEST_HAS_PARALLEL /* If static client must wait for server to write config file */ @@ -582,25 +557,11 @@ HG_Test_init(int argc, char *argv[], struct hg_test_info *hg_test_info) printf("# Target name read: %s\n", hg_test_info->na_test_info.target_name); - /* Look up target addr using target name info */ - request = hg_request_create(hg_test_info->request_class); - lookup_args.addr_ptr = &hg_test_info->target_addr; - lookup_args.request = request; - /* Forward call to remote addr and get a new request */ - ret = HG_Addr_lookup(hg_test_info->context, hg_test_addr_lookup_cb, - &lookup_args, hg_test_info->na_test_info.target_name, - HG_OP_ID_IGNORE); + ret = HG_Addr_lookup2(hg_test_info->hg_class, + hg_test_info->na_test_info.target_name, &hg_test_info->target_addr); HG_TEST_CHECK_HG_ERROR(done, ret, "HG_Addr_lookup() failed (%s)", HG_Error_to_string(ret)); - - /* Wait for request to be marked completed */ - hg_request_wait(request, HG_MAX_IDLE_TIME, &flag); - HG_TEST_CHECK_ERROR(flag == 0, done, ret, HG_TIMEOUT, - "Operation did not complete"); - - /* Free request */ - hg_request_destroy(request); } done: From f3bff206e1c5edbb6174a6a9bf9af614fefd51c9 Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Mon, 1 Jun 2020 23:42:54 -0500 Subject: [PATCH 11/30] NA SM: rework plugin to use connectionless model Use immediate lookup and remove connection handshake Use hash table for address caching Switch to SOCK_DGRAM unix sockets Use single shared segment with queue pairs instead of separate segments Currently limit number of SM peers to 256 Use atomic shared queue pair reservation Use shared command queue instead of socket when using NA_NO_BLOCK Rework polling/progress routines Improve SM error handling --- src/mercury_core_types.h | 1 + src/na/na_sm.c | 4627 +++++++++++++++++++++----------------- src/na/na_types.h | 1 + 3 files changed, 2586 insertions(+), 2043 deletions(-) diff --git a/src/mercury_core_types.h b/src/mercury_core_types.h index e591f18a..8752aa30 100644 --- a/src/mercury_core_types.h +++ b/src/mercury_core_types.h @@ -41,6 +41,7 @@ struct hg_init_info { X(HG_ACCESS) /*!< permission denied */ \ X(HG_FAULT) /*!< bad address */ \ X(HG_BUSY) /*!< device or resource busy */ \ + X(HG_EXIST) /*!< entry already exists */ \ X(HG_NODEV) /*!< no such device */ \ X(HG_INVALID_ARG) /*!< invalid argument */ \ X(HG_PROTOCOL_ERROR) /*!< protocol error */ \ diff --git a/src/na/na_sm.c b/src/na/na_sm.c index 71333d99..7933be68 100644 --- a/src/na/na_sm.c +++ b/src/na/na_sm.c @@ -9,209 +9,248 @@ */ #if !defined(_WIN32) && !defined(_GNU_SOURCE) -#define _GNU_SOURCE +# define _GNU_SOURCE #endif #include "na_plugin.h" -#include "mercury_thread_spin.h" -#include "mercury_time.h" -#include "mercury_poll.h" #include "mercury_event.h" +#include "mercury_hash_table.h" +#include "mercury_list.h" #include "mercury_mem.h" +#include "mercury_poll.h" +#include "mercury_queue.h" +#include "mercury_thread_rwlock.h" +#include "mercury_thread_spin.h" +#include "mercury_time.h" +#include +#include #include #include -#include #ifdef _WIN32 -# include +# include #else -# include -# include -# include -# include -# include -# include -# include -# include -# include -# if defined(NA_SM_HAS_CMA) -# include -# include -# elif defined(__APPLE__) -# include -# include -# endif +# include +# include +# include +# include +# include +# include +# include +# include +# include +# if defined(NA_SM_HAS_CMA) +# include +# include +# elif defined(__APPLE__) +# include +# include +# endif #endif /****************/ /* Local Macros */ /****************/ -/* Plugin constants */ -#define NA_SM_MAX_FILENAME 64 -#define NA_SM_NUM_BUFS 64 -#define NA_SM_CACHE_LINE_SIZE HG_UTIL_CACHE_ALIGNMENT -#define NA_SM_RING_BUF_SIZE \ - (sizeof(struct na_sm_ring_buf) + NA_SM_NUM_BUFS * HG_ATOMIC_QUEUE_ELT_SIZE) -#define NA_SM_COPY_BUF_SIZE 4096 -#define NA_SM_CLEANUP_NFDS 16 +/* Default cache line size */ +#define NA_SM_CACHE_LINE_SIZE HG_MEM_CACHE_LINE_SIZE -#define NA_SM_LISTEN_BACKLOG 64 -#define NA_SM_ACCEPT_INTERVAL 100 /* 100 ms */ +/* Default page size */ +#define NA_SM_PAGE_SIZE HG_MEM_PAGE_SIZE + +/* Default filenames/paths */ +#define NA_SM_SHM_PATH "/dev/shm" +#define NA_SM_SOCK_NAME "/sock" + +/* Max filename length used for shared files */ +#define NA_SM_MAX_FILENAME 64 + +/* Max number of shared-memory buffers (reserved by 64-bit atomic integer) */ +#define NA_SM_NUM_BUFS 64 + +/* Size of shared-memory buffer */ +#define NA_SM_COPY_BUF_SIZE NA_SM_PAGE_SIZE + +/* Max number of fds used for cleanup */ +#define NA_SM_CLEANUP_NFDS 16 + +/* Max number of peers */ +#define NA_SM_MAX_PEERS (NA_CONTEXT_ID_MAX + 1) /* Msg sizes */ -#define NA_SM_UNEXPECTED_SIZE NA_SM_COPY_BUF_SIZE -#define NA_SM_EXPECTED_SIZE NA_SM_UNEXPECTED_SIZE +#define NA_SM_UNEXPECTED_SIZE NA_SM_COPY_BUF_SIZE +#define NA_SM_EXPECTED_SIZE NA_SM_UNEXPECTED_SIZE /* Max tag */ -#define NA_SM_MAX_TAG NA_TAG_UB +#define NA_SM_MAX_TAG NA_TAG_MAX + +/* Max events */ +#define NA_SM_MAX_EVENTS 16 /* Op ID status bits */ -#define NA_SM_OP_COMPLETED (1 << 0) -#define NA_SM_OP_CANCELED (1 << 1) -#define NA_SM_OP_QUEUED (1 << 2) +#define NA_SM_OP_COMPLETED (1 << 0) +#define NA_SM_OP_CANCELED (1 << 1) +#define NA_SM_OP_QUEUED (1 << 2) /* Private data access */ -#define NA_SM_CLASS(na_class) \ - ((struct na_sm_class *)(na_class->plugin_class)) +#define NA_SM_CLASS(na_class) ((struct na_sm_class *) (na_class->plugin_class)) -/* Min macro */ -#define NA_SM_MIN(a, b) \ - (a < b) ? a : b +/* Generate SHM file name */ +#define NA_SM_GEN_SHM_NAME(filename, maxlen, username, pid, id) \ + snprintf( \ + filename, maxlen, "%s_%s-%d-%u", NA_SM_SHM_PREFIX, username, pid, id) -/* Struct msghdr initializer */ -#define NA_SM_MSGHDR_INITIALIZER {NULL, 0, NULL, 0, NULL, 0, 0} - -/* Default filenames/paths */ -#define NA_SM_SHM_PATH "/dev/shm" - -#define NA_SM_GEN_SHM_NAME(filename, username, na_sm_addr) \ - do { \ - sprintf(filename, "%s_%s-%d-%u", NA_SM_SHM_PREFIX, \ - username, na_sm_addr->pid, na_sm_addr->id); \ - } while (0) - -#define NA_SM_GEN_SOCK_PATH(pathname, username, na_sm_addr) \ - do { \ - sprintf(pathname, "%s/%s_%s/%d/%u", NA_SM_TMP_DIRECTORY, \ - NA_SM_SHM_PREFIX, username, na_sm_addr->pid, na_sm_addr->id); \ - } while (0) - -#define NA_SM_SEND_NAME "s" /* used for pair_name */ -#define NA_SM_RECV_NAME "r" /* used for pair_name */ -#define NA_SM_GEN_RING_NAME(filename, pair_name, username, na_sm_addr) \ - do { \ - sprintf(filename, "%s_%s-%d-%u-%u-" pair_name, NA_SM_SHM_PREFIX, \ - username, na_sm_addr->pid, na_sm_addr->id, na_sm_addr->conn_id);\ - } while (0) +/* Generate socket path */ +#define NA_SM_GEN_SOCK_PATH(pathname, maxlen, username, pid, id) \ + snprintf(pathname, maxlen, "%s/%s_%s/%d/%u", NA_SM_TMP_DIRECTORY, \ + NA_SM_SHM_PREFIX, username, pid, id); #ifndef HG_UTIL_HAS_SYSEVENTFD_H -#define NA_SM_GEN_FIFO_NAME(filename, pair_name, username, na_sm_addr) \ - do { \ - sprintf(filename, "%s/%s_%s/%d/%u/fifo-%u-" pair_name, \ - NA_SM_TMP_DIRECTORY, NA_SM_SHM_PREFIX, username, \ - na_sm_addr->pid, na_sm_addr->id, na_sm_addr->conn_id); \ - } while (0) +# define NA_SM_GEN_FIFO_NAME( \ + filename, maxlen, username, pid, id, index, pair) \ + snprintf(filename, maxlen, "%s/%s_%s/%d/%u/fifo-%u-%c", \ + NA_SM_TMP_DIRECTORY, NA_SM_SHM_PREFIX, username, pid, id, index, \ + pair) #endif /************************************/ /* Local Type and Struct Definition */ /************************************/ +/* Msg header */ typedef union { - hg_atomic_int32_t val; - char pad[NA_SM_CACHE_LINE_SIZE]; -} na_sm_cacheline_atomic_int32_t; + struct { + unsigned int tag : 32; /* Message tag : UINT MAX */ + unsigned int buf_size : 16; /* Buffer length: 4KB MAX */ + unsigned int buf_idx : 8; /* Index reserved: 64 MAX */ + unsigned int type : 8; /* Message type */ + } hdr; + na_uint64_t val; +} na_sm_msg_hdr_t; +/* Make sure this is cache-line aligned */ typedef union { hg_atomic_int64_t val; char pad[NA_SM_CACHE_LINE_SIZE]; } na_sm_cacheline_atomic_int64_t; typedef union { - struct { - unsigned int type : 4; /* Message type */ - unsigned int buf_idx : 8; /* Index reserved: 64 MAX */ - unsigned int buf_size : 16; /* Buffer length: 4KB MAX */ - unsigned int tag : 32; /* Message tag : UINT MAX */ - unsigned int pad : 4; /* 4 bits left */ - } hdr; - na_uint64_t val; -} na_sm_cacheline_hdr_t; + hg_atomic_int64_t val[4]; + char pad[NA_SM_CACHE_LINE_SIZE]; +} na_sm_cacheline_atomic_int256_t; -/* Ring buffer */ -struct na_sm_ring_buf { - struct hg_atomic_queue queue; - char pad[NA_SM_COPY_BUF_SIZE - sizeof(struct hg_atomic_queue) - - NA_SM_NUM_BUFS * HG_ATOMIC_QUEUE_ELT_SIZE]; +/* Msg buffers (page aligned) */ +struct na_sm_copy_buf { + char buf[NA_SM_NUM_BUFS][NA_SM_COPY_BUF_SIZE]; /* Array of buffers */ + hg_thread_spin_t buf_locks[NA_SM_NUM_BUFS]; /* Locks on buffers */ + na_sm_cacheline_atomic_int64_t available; /* Available bitmask */ }; -/* Shared copy buffer */ -struct na_sm_copy_buf { - na_sm_cacheline_atomic_int64_t available; /* Atomic bitmask */ - char buf[NA_SM_NUM_BUFS][NA_SM_COPY_BUF_SIZE]; /* Buffer used for msgs */ - char pad[NA_SM_COPY_BUF_SIZE - NA_SM_CACHE_LINE_SIZE]; +/* Msg queue (allocate queue's flexible array member statically) */ +struct na_sm_msg_queue { + hg_atomic_int32_t prod_head; + hg_atomic_int32_t prod_tail; + unsigned int prod_size; + unsigned int prod_mask; + hg_util_uint64_t drops; + hg_atomic_int32_t cons_head + __attribute__((aligned(HG_MEM_CACHE_LINE_SIZE))); + hg_atomic_int32_t cons_tail; + unsigned int cons_size; + unsigned int cons_mask; + hg_atomic_int64_t ring[NA_SM_NUM_BUFS] + __attribute__((aligned(HG_MEM_CACHE_LINE_SIZE))); }; -/* Poll type */ -typedef enum na_sm_poll_type { - NA_SM_ACCEPT = 1, - NA_SM_SOCK, - NA_SM_NOTIFY -} na_sm_poll_type_t; - -/* Poll data */ -struct na_sm_poll_data { - na_class_t *na_class; - struct na_sm_addr *addr; /* Address */ - na_sm_poll_type_t type; /* Type of operation */ +/* Shared queue pair */ +struct na_sm_queue_pair { + struct na_sm_msg_queue tx_queue; /* Send queue */ + struct na_sm_msg_queue rx_queue; /* Recv queue */ +}; + +/* Cmd values */ +typedef enum { NA_SM_RESERVED = 1, NA_SM_RELEASED } na_sm_cmd_t; + +/* Cmd header */ +typedef union { + struct { + unsigned int pid : 32; /* PID */ + unsigned int id : 8; /* ID */ + unsigned int pair_idx : 8; /* Index reserved */ + unsigned int type : 8; /* Cmd type */ + unsigned int pad : 8; /* 8 bits left */ + } hdr; + na_uint64_t val; +} na_sm_cmd_hdr_t; + +/* Cmd queue (allocate queue's flexible array member statically) */ +struct na_sm_cmd_queue { + hg_atomic_int32_t prod_head; + hg_atomic_int32_t prod_tail; + unsigned int prod_size; + unsigned int prod_mask; + hg_util_uint64_t drops; + hg_atomic_int32_t cons_head + __attribute__((aligned(HG_MEM_CACHE_LINE_SIZE))); + hg_atomic_int32_t cons_tail; + unsigned int cons_size; + unsigned int cons_mask; + /* To be safe, make the queue twice as large */ + hg_atomic_int64_t ring[NA_SM_MAX_PEERS * 2] + __attribute__((aligned(HG_MEM_CACHE_LINE_SIZE))); }; -/* Sock progress type */ -typedef enum { - NA_SM_ADDR_INFO, - NA_SM_CONN_ID, - NA_SM_SOCK_DONE -} na_sm_sock_progress_t; +/* Shared region */ +struct na_sm_region { + struct na_sm_copy_buf copy_bufs; /* Pool of msg buffers */ + struct na_sm_queue_pair queue_pairs[NA_SM_MAX_PEERS] + __attribute__((aligned(NA_SM_PAGE_SIZE))); /* Msg queue pairs */ + struct na_sm_cmd_queue cmd_queue; /* Cmd queue */ + na_sm_cacheline_atomic_int256_t available; /* Available pairs */ +}; /* Address */ struct na_sm_addr { - HG_QUEUE_ENTRY(na_sm_addr) entry; /* Next queue entry */ - HG_QUEUE_ENTRY(na_sm_addr) poll_entry; /* Next poll queue entry */ - struct na_sm_ring_buf *na_sm_send_ring_buf; /* Shared send ring buffer */ - struct na_sm_ring_buf *na_sm_recv_ring_buf; /* Shared recv ring buffer */ - struct na_sm_copy_buf *na_sm_copy_buf; /* Shared copy buffer */ - struct na_sm_poll_data *sock_poll_data; /* Sock poll data */ - struct na_sm_poll_data *local_notify_poll_data; /* Notify poll data */ - pid_t pid; /* PID */ - na_sm_sock_progress_t sock_progress; /* Current sock progress state */ - unsigned int id; /* SM ID */ - unsigned int conn_id; /* Connection ID */ - int sock; /* Sock fd */ - int local_notify; /* Local notify fd */ - int remote_notify; /* Remote notify fd */ - hg_atomic_int32_t ref_count; /* Ref count */ - na_bool_t accepted; /* Created on accept */ - na_bool_t self; /* Self address */ + HG_LIST_ENTRY(na_sm_addr) entry; /* Entry in poll list */ + struct na_sm_region *shared_region; /* Shared-memory region */ + struct na_sm_msg_queue *tx_queue; /* Pointer to shared tx queue */ + struct na_sm_msg_queue *rx_queue; /* Pointer to shared rx queue */ + int tx_notify; /* Notify fd for tx queue */ + int rx_notify; /* Notify fd for rx queue */ + hg_atomic_int32_t ref_count; /* Ref count */ + pid_t pid; /* PID */ + na_uint8_t id; /* SM ID */ + na_uint8_t queue_pair_idx; /* Shared queue pair index */ + na_bool_t unexpected; /* Unexpected address */ }; -/* Memory handle */ -struct na_sm_mem_handle { - struct iovec *iov; - unsigned long iovcnt; - unsigned long flags; /* Flag of operation access */ - size_t len; +/* Address list */ +struct na_sm_addr_list { + HG_LIST_HEAD(na_sm_addr) list; + hg_thread_spin_t lock; }; +/* Map (used to cache addresses) */ +struct na_sm_map { + hg_thread_rwlock_t lock; + hg_hash_table_t *map; +}; -/* Unexpected message info */ -struct na_sm_unexpected_info { - HG_QUEUE_ENTRY(na_sm_unexpected_info) entry; - struct na_sm_addr *na_sm_addr; - void *buf; - na_size_t buf_size; - na_tag_t tag; +/* Map insert cb args */ +struct na_sm_lookup_args { + struct na_sm_endpoint *endpoint; + const char *username; + pid_t pid; + na_uint8_t id; +}; + +/* Memory handle */ +struct na_sm_mem_handle { + struct iovec *iov; /* I/O segments */ + unsigned long iovcnt; /* Segment count */ + size_t len; /* Size of region */ + na_uint8_t flags; /* Flag of operation access */ }; /* Msg info */ @@ -225,43 +264,62 @@ struct na_sm_msg_info { na_tag_t tag; }; +/* Unexpected msg info */ +struct na_sm_unexpected_info { + HG_QUEUE_ENTRY(na_sm_unexpected_info) entry; + struct na_sm_addr *na_sm_addr; + void *buf; + na_size_t buf_size; + na_tag_t tag; +}; + +/* Unexpected msg queue */ +struct na_sm_unexpected_msg_queue { + HG_QUEUE_HEAD(na_sm_unexpected_info) queue; + hg_thread_spin_t lock; +}; + /* Operation ID */ struct na_sm_op_id { struct na_cb_completion_data completion_data; /* Completion data */ union { struct na_sm_msg_info msg; - } info; /* Op info */ - HG_QUEUE_ENTRY(na_sm_op_id) entry; /* Entry in queue */ - na_class_t *na_class; /* NA class associated */ - na_context_t *context; /* NA context associated */ - struct na_sm_addr *na_sm_addr; /* Address associated */ - hg_atomic_int32_t status; /* Operation status */ - hg_atomic_int32_t ref_count; /* Refcount */ + } info; /* Op info */ + HG_QUEUE_ENTRY(na_sm_op_id) entry; /* Entry in queue */ + na_class_t *na_class; /* NA class associated */ + na_context_t *context; /* NA context associated */ + struct na_sm_addr *na_sm_addr; /* Address associated */ + hg_atomic_int32_t status; /* Operation status */ + hg_atomic_int32_t ref_count; /* Refcount */ +}; + +/* Op ID queue */ +struct na_sm_op_queue { + HG_QUEUE_HEAD(na_sm_op_id) queue; + hg_thread_spin_t lock; +}; + +/* Endpoint */ +struct na_sm_endpoint { + struct na_sm_map addr_map; /* Address map */ + struct na_sm_unexpected_msg_queue + unexpected_msg_queue; /* Unexpected msg queue */ + struct na_sm_op_queue unexpected_op_queue; /* Unexpected op queue */ + struct na_sm_op_queue expected_op_queue; /* Expected op queue */ + struct na_sm_op_queue retry_op_queue; /* Retry op queue */ + struct na_sm_addr_list poll_addr_list; /* List of addresses to poll */ + struct na_sm_addr *source_addr; /* Source addr */ + hg_poll_set_t *poll_set; /* Poll set */ + int sock; /* Sock fd */ + na_bool_t listen; /* Listen on sock */ }; /* Private data */ struct na_sm_class { - HG_QUEUE_HEAD(na_sm_addr) accepted_addr_queue; - HG_QUEUE_HEAD(na_sm_addr) poll_addr_queue; - HG_QUEUE_HEAD(na_sm_unexpected_info) unexpected_msg_queue; - HG_QUEUE_HEAD(na_sm_op_id) lookup_op_queue; - HG_QUEUE_HEAD(na_sm_op_id) unexpected_op_queue; - HG_QUEUE_HEAD(na_sm_op_id) expected_op_queue; - HG_QUEUE_HEAD(na_sm_op_id) retry_op_queue; - hg_time_t last_accept_time; - char *username; - struct na_sm_addr *self_addr; - hg_poll_set_t *poll_set; - hg_thread_spin_t accepted_addr_queue_lock; - hg_thread_spin_t poll_addr_queue_lock; - hg_thread_spin_t unexpected_msg_queue_lock; - hg_thread_spin_t lookup_op_queue_lock; - hg_thread_spin_t unexpected_op_queue_lock; - hg_thread_spin_t expected_op_queue_lock; - hg_thread_spin_t retry_op_queue_lock; - hg_thread_spin_t copy_buf_lock; - na_bool_t no_wait; - na_bool_t no_retry; + struct na_sm_endpoint endpoint; /* Endpoint */ + char *username; /* Username */ + na_uint8_t max_contexts; /* Max number of contexts */ + na_bool_t no_wait; /* Ignore wait object */ }; /********************/ @@ -275,702 +333,599 @@ struct na_sm_class { static char * getlogin_safe(void); +/** + * Convert errno to NA return values. + */ +static na_return_t +na_sm_errno_to_na(int rc); /** - * Open shared buf. + * Map shared-memory object. */ static void * -na_sm_open_shared_buf( - const char *name, - size_t buf_size, - na_bool_t create - ); +na_sm_shm_map(const char *name, na_size_t length, na_bool_t create); + +/** + * Unmap shared-memory object. + */ +static na_return_t +na_sm_shm_unmap(const char *name, void *addr, na_size_t length); + +/** + * Clean up dangling shm segments. + */ +static int +na_sm_shm_cleanup( + const char *fpath, const struct stat *sb, int typeflag, struct FTW *ftwbuf); + +/** + * Initialize queue. + */ +static void +na_sm_msg_queue_init(struct na_sm_msg_queue *na_sm_queue); + +/** + * Multi-producer enqueue. + */ +static NA_INLINE na_bool_t +na_sm_msg_queue_push( + struct na_sm_msg_queue *na_sm_queue, na_sm_msg_hdr_t msg_hdr); + +/** + * Multi-consumer dequeue. + */ +static NA_INLINE na_bool_t +na_sm_msg_queue_pop( + struct na_sm_msg_queue *na_sm_msg_queue, na_sm_msg_hdr_t *msg_hdr_ptr); + +/** + * Check whether queue is empty. + */ +static NA_INLINE na_bool_t +na_sm_msg_queue_is_empty(struct na_sm_msg_queue *na_sm_queue); + +/** + * Initialize queue. + */ +static void +na_sm_cmd_queue_init(struct na_sm_cmd_queue *na_sm_queue); + +/** + * Multi-producer enqueue. + */ +static NA_INLINE na_bool_t +na_sm_cmd_queue_push( + struct na_sm_cmd_queue *na_sm_queue, na_sm_cmd_hdr_t cmd_hdr); + +/** + * Multi-consumer dequeue. + */ +static NA_INLINE na_bool_t +na_sm_cmd_queue_pop( + struct na_sm_cmd_queue *na_sm_queue, na_sm_cmd_hdr_t *cmd_hdr_ptr); + +/** + * Check whether queue is empty. + */ +static NA_INLINE na_bool_t +na_sm_cmd_queue_is_empty(struct na_sm_cmd_queue *na_sm_queue); + +/** + * Generate key for addr map. + */ +static NA_INLINE na_uint64_t +na_sm_addr_to_key(pid_t pid, na_uint8_t id); + +/** + * Key hash for hash table. + */ +static NA_INLINE unsigned int +na_sm_addr_key_hash(hg_hash_table_key_t vlocation); + +/** + * Compare key. + */ +static NA_INLINE int +na_sm_addr_key_equal( + hg_hash_table_key_t vlocation1, hg_hash_table_key_t vlocation2); /** - * Close shared buf. + * Get SM address from string. */ static na_return_t -na_sm_close_shared_buf( - const char *filename, - void *buf, - size_t buf_size - ); +na_sm_string_to_addr(const char *str, pid_t *pid, na_uint8_t *id); /** - * Create UNIX domain socket. + * Open shared-memory region. */ static na_return_t -na_sm_create_sock( - const char *pathname, - na_bool_t na_listen, - int *sock); +na_sm_region_open(const char *username, pid_t pid, na_uint8_t id, + na_bool_t create, struct na_sm_region **region); /** - * Close socket. + * Close shared-memory region. */ static na_return_t -na_sm_close_sock( - int sock, - const char *pathname - ); +na_sm_region_close(const char *username, pid_t pid, na_uint8_t id, + na_bool_t remove, struct na_sm_region *region); /** - * Clean up file. + * Open UNIX domain socket. */ -static int -na_sm_cleanup_file( - const char *fpath, - const struct stat *sb, - int typeflag, - struct FTW *ftwbuf - ); +static na_return_t +na_sm_sock_open(const char *username, pid_t pid, na_uint8_t id, + na_bool_t create, int *sock); /** - * Clean up shm segment. + * Close socket. */ -static int -na_sm_cleanup_shm( - const char *fpath, - const struct stat *sb, - int typeflag, - struct FTW *ftwbuf - ); +static na_return_t +na_sm_sock_close( + const char *username, pid_t pid, na_uint8_t id, na_bool_t remove, int sock); -#ifndef HG_UTIL_HAS_SYSEVENTFD_H +/** + * Create tmp path for UNIX socket. + */ +static na_return_t +na_sm_sock_path_create(const char *pathname); + +/** + * Remove tmp path for UNIX socket. + */ +static na_return_t +na_sm_sock_path_remove(const char *pathname); /** - * Create event using named pipe. + * Clean up tmp paths for UNIX socket. */ static int -na_sm_event_create( - const char *filename - ); +na_sm_sock_path_cleanup( + const char *fpath, const struct stat *sb, int typeflag, struct FTW *ftwbuf); + +/** + * Create event. + */ +static na_return_t +na_sm_event_create(const char *username, pid_t pid, na_uint8_t id, + na_uint8_t pair_index, unsigned char pair, int *event); /** * Destroy event. */ static na_return_t -na_sm_event_destroy( - const char *filename, - int fd - ); +na_sm_event_destroy(const char *username, pid_t pid, na_uint8_t id, + na_uint8_t pair_index, unsigned char pair, na_bool_t remove, int event); /** * Set event. */ -static na_return_t -na_sm_event_set( - int fd - ); +static NA_INLINE na_return_t +na_sm_event_set(int event); /** * Get event. */ -static na_return_t -na_sm_event_get( - int fd, - na_bool_t *signaled - ); - -#endif +static NA_INLINE na_return_t +na_sm_event_get(int event, na_bool_t *signaled); /** * Register addr to poll set. */ static na_return_t na_sm_poll_register( - na_class_t *na_class, - na_sm_poll_type_t poll_type, - struct na_sm_addr *na_sm_addr - ); + hg_poll_set_t *poll_set, int fd, hg_poll_cb_t poll_cb, void *poll_arg); /** * Deregister addr from poll set. */ static na_return_t -na_sm_poll_deregister( - na_class_t *na_class, - na_sm_poll_type_t poll_type, - struct na_sm_addr *na_sm_addr - ); +na_sm_poll_deregister(hg_poll_set_t *poll_set, int fd); /** - * Create copy buf and sock and register self address. + * Open shared-memory endpoint. */ -static na_return_t na_sm_setup_shm( - na_class_t *na_class, - struct na_sm_addr *na_sm_addr - ); +static na_return_t +na_sm_endpoint_open(struct na_sm_endpoint *na_sm_endpoint, const char *username, + pid_t pid, na_uint8_t id, na_bool_t listen, na_bool_t no_wait); /** - * Send addr info. + * Close shared-memory endpoint. */ static na_return_t -na_sm_send_addr_info( - na_class_t *na_class, - struct na_sm_addr *na_sm_addr - ); +na_sm_endpoint_close( + struct na_sm_endpoint *na_sm_endpoint, const char *username); /** - * Recv addr info. + * Reserve queue pair. */ -static na_return_t -na_sm_recv_addr_info( - struct na_sm_addr *na_sm_addr, - na_bool_t *received - ); +static NA_INLINE na_return_t +na_sm_queue_pair_reserve(struct na_sm_region *na_sm_region, na_uint8_t *index); + +/** + * Release queue pair. + */ +static NA_INLINE void +na_sm_queue_pair_release(struct na_sm_region *na_sm_region, na_uint8_t index); /** - * Send connection ID. + * Lookup addr key from map. + */ +static NA_INLINE struct na_sm_addr * +na_sm_addr_map_lookup(struct na_sm_map *na_sm_map, na_uint64_t key); + +/** + * Insert new addr key into map. Execute callback while write lock is acquired. */ static na_return_t -na_sm_send_conn_id( - struct na_sm_addr *na_sm_addr - ); +na_sm_addr_map_insert(struct na_sm_map *na_sm_map, na_uint64_t key, + na_return_t (*insert_cb)(void *, struct na_sm_addr **), void *arg, + struct na_sm_addr **addr); /** - * Recv connection ID. + * Reserve new queue pair and send event signals to target. */ static na_return_t -na_sm_recv_conn_id( - struct na_sm_addr *na_sm_addr, - na_bool_t *received - ); +na_sm_addr_lookup_insert_cb(void *arg, struct na_sm_addr **addr); /** - * Initialize ring buffer. + * Create new address. */ -static void -na_sm_ring_buf_init( - struct na_sm_ring_buf *na_sm_ring_buf - ); +static na_return_t +na_sm_addr_create(struct na_sm_endpoint *na_sm_endpoint, + struct na_sm_region *shared_region, pid_t pid, na_uint8_t id, + na_uint8_t queue_pair_idx, int tx_notify, int rx_notify, + na_bool_t unexpected, struct na_sm_addr **addr); /** - * Multi-producer safe lock-free ring buffer enqueue. + * Destroy address. */ -static NA_INLINE na_bool_t -na_sm_ring_buf_push( - struct na_sm_ring_buf *na_sm_ring_buf, - na_sm_cacheline_hdr_t na_sm_hdr - ); +static na_return_t +na_sm_addr_destroy(struct na_sm_endpoint *na_sm_endpoint, const char *username, + struct na_sm_addr *na_sm_addr); /** - * Multi-consumer dequeue. + * Send events as ancillary data. */ -static NA_INLINE na_bool_t -na_sm_ring_buf_pop( - struct na_sm_ring_buf *na_sm_ring_buf, - na_sm_cacheline_hdr_t *na_sm_hdr_ptr - ); +static na_return_t +na_sm_addr_event_send(int sock, const char *username, pid_t pid, na_uint8_t id, + na_sm_cmd_hdr_t cmd_hdr, int tx_notify, int rx_notify, + na_bool_t ignore_error); /** - * Check whether queue is empty. + * Recv events as ancillary data. */ -static NA_INLINE na_bool_t -na_sm_ring_buf_is_empty( - struct na_sm_ring_buf *na_sm_ring_buf - ); +static na_return_t +na_sm_addr_event_recv(int sock, na_sm_cmd_hdr_t *cmd_hdr, int *tx_notify, + int *rx_notify, na_bool_t *received); /** - * Reserve shared copy buf. + * Reserve shared buffer. */ static NA_INLINE na_return_t -na_sm_reserve_and_copy_buf( - na_class_t *na_class, - struct na_sm_copy_buf *na_sm_copy_buf, - const void *buf, - size_t buf_size, - unsigned int *idx_reserved - ); +na_sm_buf_reserve(struct na_sm_copy_buf *na_sm_copy_buf, unsigned int *index); /** - * Free and copy buf. + * Release shared buffer. */ static NA_INLINE void -na_sm_copy_and_free_buf( - na_class_t *na_class, - struct na_sm_copy_buf *na_sm_copy_buf, - void *buf, - size_t buf_size, - unsigned int idx_reserved - ); +na_sm_buf_release(struct na_sm_copy_buf *na_sm_copy_buf, unsigned int index); /** - * Release shared copy buf. + * Copy src to shared buffer. */ static NA_INLINE void -na_sm_release_buf( - struct na_sm_copy_buf *na_sm_copy_buf, - unsigned int idx_reserved - ); +na_sm_buf_copy_to(struct na_sm_copy_buf *na_sm_copy_buf, unsigned int index, + const void *src, size_t n); /** - * Insert message header into ring buffer. + * Copy from shared buffer to dest. */ -static na_return_t -na_sm_msg_insert( - na_class_t *na_class, - struct na_sm_op_id *na_sm_op_id, - unsigned int idx_reserved - ); +static NA_INLINE void +na_sm_buf_copy_from(struct na_sm_copy_buf *na_sm_copy_buf, unsigned int index, + void *dest, size_t n); /** * Translate offset from mem_handle into usable iovec. */ static void -na_sm_offset_translate( - struct na_sm_mem_handle *mem_handle, - na_offset_t offset, - na_size_t length, - struct iovec *iov, - unsigned long *iovcnt - ); +na_sm_offset_translate(struct na_sm_mem_handle *mem_handle, na_offset_t offset, + na_size_t length, struct iovec *iov, unsigned long *iovcnt); /** - * Progress callback. + * Progress on endpoint sock. */ static int -na_sm_progress_cb( - void *arg, - int error, - hg_util_bool_t *progressed - ); +na_sm_progress_sock(void *arg, int error, struct hg_poll_event *event); /** - * Progress error. + * Process cmd. */ static na_return_t -na_sm_progress_error( - na_class_t *na_class, - struct na_sm_addr *poll_addr - ); +na_sm_process_cmd(struct na_sm_endpoint *na_sm_endpoint, const char *username, + na_sm_cmd_hdr_t cmd_hdr, int tx_notify, int rx_notify); /** - * Progress on accept. + * Progress on tx notifications. */ -static na_return_t -na_sm_progress_accept( - na_class_t *na_class, - struct na_sm_addr *poll_addr, - na_bool_t *progressed - ); +static int +na_sm_progress_tx_notify(void *arg, int error, struct hg_poll_event *event); /** - * Progress on socket. + * Progress on rx notifications. */ -static na_return_t -na_sm_progress_sock( - na_class_t *na_class, - struct na_sm_addr *poll_addr, - na_bool_t *progressed - ); +static int +na_sm_progress_rx_notify(void *arg, int error, struct hg_poll_event *event); /** - * Progress on notifications. + * Progress rx queue. */ static na_return_t -na_sm_progress_notify( - na_class_t *na_class, - struct na_sm_addr *poll_addr, - na_bool_t *progressed - ); +na_sm_progress_rx_queue(struct na_sm_endpoint *na_sm_endpoint, + struct na_sm_addr *poll_addr, na_bool_t *progressed); /** - * Progress on unexpected messages. + * Process unexpected messages. */ static na_return_t -na_sm_progress_unexpected( - na_class_t *na_class, - struct na_sm_addr *poll_addr, - na_sm_cacheline_hdr_t na_sm_hdr - ); +na_sm_process_unexpected(struct na_sm_op_queue *unexpected_op_queue, + struct na_sm_addr *poll_addr, na_sm_msg_hdr_t msg_hdr, + struct na_sm_unexpected_msg_queue *unexpected_msg_queue); /** - * Progress on expected messages. + * Process expected messages. */ static na_return_t -na_sm_progress_expected( - na_class_t *na_class, - struct na_sm_addr *poll_addr, - na_sm_cacheline_hdr_t na_sm_hdr - ); +na_sm_process_expected(struct na_sm_op_queue *expected_op_queue, + struct na_sm_addr *poll_addr, na_sm_msg_hdr_t msg_hdr); /** - * Progress retries. + * Process retries. */ static na_return_t -na_sm_progress_retries( - na_class_t *na_class - ); +na_sm_process_retries(struct na_sm_op_queue *retry_op_queue); /** * Complete operation. */ static na_return_t -na_sm_complete( - struct na_sm_op_id *na_sm_op_id - ); +na_sm_complete(struct na_sm_op_id *na_sm_op_id, int notify); /** * Release memory. */ static NA_INLINE void -na_sm_release( - void *arg - ); +na_sm_release(void *arg); /* check_protocol */ static na_bool_t -na_sm_check_protocol( - const char *protocol_name - ); +na_sm_check_protocol(const char *protocol_name); /* initialize */ static na_return_t na_sm_initialize( - na_class_t *na_class, - const struct na_info *na_info, - na_bool_t listen - ); + na_class_t *na_class, const struct na_info *na_info, na_bool_t listen); /* finalize */ static na_return_t -na_sm_finalize( - na_class_t *na_class - ); +na_sm_finalize(na_class_t *na_class); /* cleanup */ static void -na_sm_cleanup( - void - ); +na_sm_cleanup(void); /* op_create */ static na_op_id_t -na_sm_op_create( - na_class_t *na_class - ); +na_sm_op_create(na_class_t *na_class); /* op_destroy */ static na_return_t -na_sm_op_destroy( - na_class_t *na_class, - na_op_id_t op_id - ); +na_sm_op_destroy(na_class_t *na_class, na_op_id_t op_id); /* addr_lookup */ static na_return_t -na_sm_addr_lookup( - na_class_t *na_class, - na_context_t *context, - na_cb_t callback, - void *arg, - const char *name, - na_op_id_t *op_id - ); +na_sm_addr_lookup(na_class_t *na_class, const char *name, na_addr_t *addr); /* addr_free */ static na_return_t -na_sm_addr_free( - na_class_t *na_class, - na_addr_t addr - ); +na_sm_addr_free(na_class_t *na_class, na_addr_t addr); /* addr_self */ static na_return_t -na_sm_addr_self( - na_class_t *na_class, - na_addr_t *addr - ); +na_sm_addr_self(na_class_t *na_class, na_addr_t *addr); /* addr_dup */ static na_return_t -na_sm_addr_dup( - na_class_t *na_class, - na_addr_t addr, - na_addr_t *new_addr - ); +na_sm_addr_dup(na_class_t *na_class, na_addr_t addr, na_addr_t *new_addr); /* addr_cmp */ static na_bool_t -na_sm_addr_cmp( - na_class_t *na_class, - na_addr_t addr1, - na_addr_t addr2 - ); +na_sm_addr_cmp(na_class_t *na_class, na_addr_t addr1, na_addr_t addr2); /* addr_is_self */ static NA_INLINE na_bool_t -na_sm_addr_is_self( - na_class_t *na_class, - na_addr_t addr - ); +na_sm_addr_is_self(na_class_t *na_class, na_addr_t addr); /* addr_to_string */ static na_return_t na_sm_addr_to_string( - na_class_t *na_class, - char *buf, - na_size_t *buf_size, - na_addr_t addr - ); + na_class_t *na_class, char *buf, na_size_t *buf_size, na_addr_t addr); + +/* addr_get_serialize_size */ +static NA_INLINE na_size_t +na_sm_addr_get_serialize_size(na_class_t *na_class, na_addr_t addr); + +/* addr_serialize */ +static na_return_t +na_sm_addr_serialize( + na_class_t *na_class, void *buf, na_size_t buf_size, na_addr_t addr); + +/* addr_deserialize */ +static na_return_t +na_sm_addr_deserialize( + na_class_t *na_class, na_addr_t *addr, const void *buf, na_size_t buf_size); /* msg_get_max_unexpected_size */ static NA_INLINE na_size_t -na_sm_msg_get_max_unexpected_size( - const na_class_t *na_class - ); +na_sm_msg_get_max_unexpected_size(const na_class_t *na_class); /* msg_get_max_expected_size */ static NA_INLINE na_size_t -na_sm_msg_get_max_expected_size( - const na_class_t *na_class - ); +na_sm_msg_get_max_expected_size(const na_class_t *na_class); /* msg_get_max_tag */ static NA_INLINE na_tag_t -na_sm_msg_get_max_tag( - const na_class_t *na_class - ); +na_sm_msg_get_max_tag(const na_class_t *na_class); /* msg_send_unexpected */ static na_return_t -na_sm_msg_send_unexpected( - na_class_t *na_class, - na_context_t *context, - na_cb_t callback, - void *arg, - const void *buf, - na_size_t buf_size, - void *plugin_data, - na_addr_t dest_addr, - na_uint8_t dest_id, - na_tag_t tag, - na_op_id_t *op_id - ); +na_sm_msg_send_unexpected(na_class_t *na_class, na_context_t *context, + na_cb_t callback, void *arg, const void *buf, na_size_t buf_size, + void *plugin_data, na_addr_t dest_addr, na_uint8_t dest_id, na_tag_t tag, + na_op_id_t *op_id); /* msg_recv_unexpected */ static na_return_t -na_sm_msg_recv_unexpected( - na_class_t *na_class, - na_context_t *context, - na_cb_t callback, - void *arg, - void *buf, - na_size_t buf_size, - void *plugin_data, - na_op_id_t *op_id - ); +na_sm_msg_recv_unexpected(na_class_t *na_class, na_context_t *context, + na_cb_t callback, void *arg, void *buf, na_size_t buf_size, + void *plugin_data, na_op_id_t *op_id); /* msg_send_expected */ static na_return_t -na_sm_msg_send_expected( - na_class_t *na_class, - na_context_t *context, - na_cb_t callback, - void *arg, - const void *buf, - na_size_t buf_size, - void *plugin_data, - na_addr_t dest_addr, - na_uint8_t dest_id, - na_tag_t tag, - na_op_id_t *op_id - ); +na_sm_msg_send_expected(na_class_t *na_class, na_context_t *context, + na_cb_t callback, void *arg, const void *buf, na_size_t buf_size, + void *plugin_data, na_addr_t dest_addr, na_uint8_t dest_id, na_tag_t tag, + na_op_id_t *op_id); /* msg_recv_expected */ static na_return_t -na_sm_msg_recv_expected( - na_class_t *na_class, - na_context_t *context, - na_cb_t callback, - void *arg, - void *buf, - na_size_t buf_size, - void *plugin_data, - na_addr_t source_addr, - na_uint8_t source_id, - na_tag_t tag, - na_op_id_t *op_id - ); +na_sm_msg_recv_expected(na_class_t *na_class, na_context_t *context, + na_cb_t callback, void *arg, void *buf, na_size_t buf_size, + void *plugin_data, na_addr_t source_addr, na_uint8_t source_id, + na_tag_t tag, na_op_id_t *op_id); /* mem_handle_create */ static na_return_t -na_sm_mem_handle_create( - na_class_t *na_class, - void *buf, - na_size_t buf_size, - unsigned long flags, - na_mem_handle_t *mem_handle - ); +na_sm_mem_handle_create(na_class_t *na_class, void *buf, na_size_t buf_size, + unsigned long flags, na_mem_handle_t *mem_handle); #ifdef NA_SM_HAS_CMA /* mem_handle_create_segments */ static na_return_t -na_sm_mem_handle_create_segments( - na_class_t *na_class, - struct na_segment *segments, - na_size_t segment_count, - unsigned long flags, - na_mem_handle_t *mem_handle - ); +na_sm_mem_handle_create_segments(na_class_t *na_class, + struct na_segment *segments, na_size_t segment_count, unsigned long flags, + na_mem_handle_t *mem_handle); #endif /* mem_handle_free */ static na_return_t -na_sm_mem_handle_free( - na_class_t *na_class, - na_mem_handle_t mem_handle - ); +na_sm_mem_handle_free(na_class_t *na_class, na_mem_handle_t mem_handle); /* mem_handle_get_serialize_size */ static NA_INLINE na_size_t na_sm_mem_handle_get_serialize_size( - na_class_t *na_class, - na_mem_handle_t mem_handle - ); + na_class_t *na_class, na_mem_handle_t mem_handle); /* mem_handle_serialize */ static na_return_t -na_sm_mem_handle_serialize( - na_class_t *na_class, - void *buf, - na_size_t buf_size, - na_mem_handle_t mem_handle - ); +na_sm_mem_handle_serialize(na_class_t *na_class, void *buf, na_size_t buf_size, + na_mem_handle_t mem_handle); /* mem_handle_deserialize */ static na_return_t -na_sm_mem_handle_deserialize( - na_class_t *na_class, - na_mem_handle_t *mem_handle, - const void *buf, - na_size_t buf_size - ); +na_sm_mem_handle_deserialize(na_class_t *na_class, na_mem_handle_t *mem_handle, + const void *buf, na_size_t buf_size); /* put */ static na_return_t -na_sm_put( - na_class_t *na_class, - na_context_t *context, - na_cb_t callback, - void *arg, - na_mem_handle_t local_mem_handle, - na_offset_t local_offset, - na_mem_handle_t remote_mem_handle, - na_offset_t remote_offset, - na_size_t length, - na_addr_t remote_addr, - na_uint8_t remote_id, - na_op_id_t *op_id - ); +na_sm_put(na_class_t *na_class, na_context_t *context, na_cb_t callback, + void *arg, na_mem_handle_t local_mem_handle, na_offset_t local_offset, + na_mem_handle_t remote_mem_handle, na_offset_t remote_offset, + na_size_t length, na_addr_t remote_addr, na_uint8_t remote_id, + na_op_id_t *op_id); /* get */ static na_return_t -na_sm_get( - na_class_t *na_class, - na_context_t *context, - na_cb_t callback, - void *arg, - na_mem_handle_t local_mem_handle, - na_offset_t local_offset, - na_mem_handle_t remote_mem_handle, - na_offset_t remote_offset, - na_size_t length, - na_addr_t remote_addr, - na_uint8_t remote_id, - na_op_id_t *op_id - ); +na_sm_get(na_class_t *na_class, na_context_t *context, na_cb_t callback, + void *arg, na_mem_handle_t local_mem_handle, na_offset_t local_offset, + na_mem_handle_t remote_mem_handle, na_offset_t remote_offset, + na_size_t length, na_addr_t remote_addr, na_uint8_t remote_id, + na_op_id_t *op_id); /* poll_get_fd */ static NA_INLINE int -na_sm_poll_get_fd( - na_class_t *na_class, - na_context_t *context - ); +na_sm_poll_get_fd(na_class_t *na_class, na_context_t *context); /* poll_try_wait */ static NA_INLINE na_bool_t -na_sm_poll_try_wait( - na_class_t *na_class, - na_context_t *context - ); +na_sm_poll_try_wait(na_class_t *na_class, na_context_t *context); /* progress */ static na_return_t na_sm_progress( - na_class_t *na_class, - na_context_t *context, - unsigned int timeout - ); + na_class_t *na_class, na_context_t *context, unsigned int timeout); /* cancel */ static na_return_t -na_sm_cancel( - na_class_t *na_class, - na_context_t *context, - na_op_id_t op_id - ); +na_sm_cancel(na_class_t *na_class, na_context_t *context, na_op_id_t op_id); /*******************/ /* Local Variables */ /*******************/ const struct na_class_ops NA_PLUGIN_OPS(sm) = { - "na", /* name */ - na_sm_check_protocol, /* check_protocol */ - na_sm_initialize, /* initialize */ - na_sm_finalize, /* finalize */ - na_sm_cleanup, /* cleanup */ - NULL, /* context_create */ - NULL, /* context_destroy */ - na_sm_op_create, /* op_create */ - na_sm_op_destroy, /* op_destroy */ - na_sm_addr_lookup, /* addr_lookup */ - NULL, /* addr_lookup2 */ - na_sm_addr_free, /* addr_free */ - NULL, /* addr_set_remove */ - na_sm_addr_self, /* addr_self */ - na_sm_addr_dup, /* addr_dup */ - na_sm_addr_cmp, /* addr_cmp */ - na_sm_addr_is_self, /* addr_is_self */ - na_sm_addr_to_string, /* addr_to_string */ - NULL, /* addr_get_serialize_size */ - NULL, /* addr_serialize */ - NULL, /* addr_deserialize */ - na_sm_msg_get_max_unexpected_size, /* msg_get_max_unexpected_size */ - na_sm_msg_get_max_expected_size, /* msg_get_max_expected_size */ - NULL, /* msg_get_unexpected_header_size */ - NULL, /* msg_get_expected_header_size */ - na_sm_msg_get_max_tag, /* msg_get_max_tag */ - NULL, /* msg_buf_alloc */ - NULL, /* msg_buf_free */ - NULL, /* msg_init_unexpected */ - na_sm_msg_send_unexpected, /* msg_send_unexpected */ - na_sm_msg_recv_unexpected, /* msg_recv_unexpected */ - NULL, /* msg_init_expected */ - na_sm_msg_send_expected, /* msg_send_expected */ - na_sm_msg_recv_expected, /* msg_recv_expected */ - na_sm_mem_handle_create, /* mem_handle_create */ + "na", /* name */ + na_sm_check_protocol, /* check_protocol */ + na_sm_initialize, /* initialize */ + na_sm_finalize, /* finalize */ + na_sm_cleanup, /* cleanup */ + NULL, /* context_create */ + NULL, /* context_destroy */ + na_sm_op_create, /* op_create */ + na_sm_op_destroy, /* op_destroy */ + na_sm_addr_lookup, /* addr_lookup */ + na_sm_addr_free, /* addr_free */ + NULL, /* addr_set_remove */ + na_sm_addr_self, /* addr_self */ + na_sm_addr_dup, /* addr_dup */ + na_sm_addr_cmp, /* addr_cmp */ + na_sm_addr_is_self, /* addr_is_self */ + na_sm_addr_to_string, /* addr_to_string */ + na_sm_addr_get_serialize_size, /* addr_get_serialize_size */ + na_sm_addr_serialize, /* addr_serialize */ + na_sm_addr_deserialize, /* addr_deserialize */ + na_sm_msg_get_max_unexpected_size, /* msg_get_max_unexpected_size */ + na_sm_msg_get_max_expected_size, /* msg_get_max_expected_size */ + NULL, /* msg_get_unexpected_header_size */ + NULL, /* msg_get_expected_header_size */ + na_sm_msg_get_max_tag, /* msg_get_max_tag */ + NULL, /* msg_buf_alloc */ + NULL, /* msg_buf_free */ + NULL, /* msg_init_unexpected */ + na_sm_msg_send_unexpected, /* msg_send_unexpected */ + na_sm_msg_recv_unexpected, /* msg_recv_unexpected */ + NULL, /* msg_init_expected */ + na_sm_msg_send_expected, /* msg_send_expected */ + na_sm_msg_recv_expected, /* msg_recv_expected */ + na_sm_mem_handle_create, /* mem_handle_create */ #ifdef NA_SM_HAS_CMA - na_sm_mem_handle_create_segments, /* mem_handle_create_segments */ + na_sm_mem_handle_create_segments, /* mem_handle_create_segments */ #else - NULL, /* mem_handle_create_segments */ + NULL, /* mem_handle_create_segments */ #endif - na_sm_mem_handle_free, /* mem_handle_free */ - NULL, /* mem_register */ - NULL, /* mem_deregister */ - NULL, /* mem_publish */ - NULL, /* mem_unpublish */ - na_sm_mem_handle_get_serialize_size, /* mem_handle_get_serialize_size */ - na_sm_mem_handle_serialize, /* mem_handle_serialize */ - na_sm_mem_handle_deserialize, /* mem_handle_deserialize */ - na_sm_put, /* put */ - na_sm_get, /* get */ - na_sm_poll_get_fd, /* poll_get_fd */ - na_sm_poll_try_wait, /* poll_try_wait */ - na_sm_progress, /* progress */ - na_sm_cancel /* cancel */ + na_sm_mem_handle_free, /* mem_handle_free */ + NULL, /* mem_register */ + NULL, /* mem_deregister */ + NULL, /* mem_publish */ + NULL, /* mem_unpublish */ + na_sm_mem_handle_get_serialize_size, /* mem_handle_get_serialize_size */ + na_sm_mem_handle_serialize, /* mem_handle_serialize */ + na_sm_mem_handle_deserialize, /* mem_handle_deserialize */ + na_sm_put, /* put */ + na_sm_get, /* get */ + na_sm_poll_get_fd, /* poll_get_fd */ + na_sm_poll_try_wait, /* poll_try_wait */ + na_sm_progress, /* progress */ + na_sm_cancel /* cancel */ }; /********************/ @@ -980,15 +935,14 @@ const struct na_class_ops NA_PLUGIN_OPS(sm) = { /* Debug information */ #ifdef NA_HAS_DEBUG static char * -itoa(uint64_t val, int base) +lltoa(hg_util_uint64_t val, char *string, int radix) { - static char buf[64] = {0}; - int i = 62; + int i = sizeof(val) * 8; - for (; val && i; --i, val /= (uint64_t) base) - buf[i] = "0123456789abcdef"[val % (uint64_t) base]; + for (; val && i; --i, val /= (hg_util_uint64_t) radix) + string[i - 1] = "0123456789abcdef"[val % (hg_util_uint64_t) radix]; - return &buf[i + 1]; + return &string[i]; } #endif @@ -1004,112 +958,409 @@ getlogin_safe(void) return passwd ? passwd->pw_name : "unknown"; } +/*---------------------------------------------------------------------------*/ +static na_return_t +na_sm_errno_to_na(int rc) +{ + na_return_t ret; + + switch (rc) { + case EPERM: + ret = NA_PERMISSION; + break; + case ENOENT: + ret = NA_NOENTRY; + break; + case EINTR: + ret = NA_INTERRUPT; + break; + case EAGAIN: + ret = NA_AGAIN; + break; + case ENOMEM: + ret = NA_NOMEM; + break; + case EACCES: + ret = NA_ACCESS; + break; + case EFAULT: + ret = NA_FAULT; + break; + case EBUSY: + ret = NA_BUSY; + break; + case EEXIST: + ret = NA_EXIST; + break; + case ENODEV: + ret = NA_NODEV; + break; + case EINVAL: + ret = NA_INVALID_ARG; + break; + case EOVERFLOW: + case ENAMETOOLONG: + ret = NA_OVERFLOW; + break; + case EMSGSIZE: + ret = NA_MSGSIZE; + break; + case EPROTONOSUPPORT: + ret = NA_PROTONOSUPPORT; + break; + case EOPNOTSUPP: + ret = NA_OPNOTSUPPORTED; + break; + case EADDRINUSE: + ret = NA_ADDRINUSE; + break; + case EADDRNOTAVAIL: + ret = NA_ADDRNOTAVAIL; + break; + case ETIMEDOUT: + ret = NA_TIMEOUT; + break; + case ECANCELED: + ret = NA_CANCELED; + break; + default: + ret = NA_PROTOCOL_ERROR; + break; + } + + return ret; +} + /*---------------------------------------------------------------------------*/ static void * -na_sm_open_shared_buf(const char *name, size_t buf_size, na_bool_t create) +na_sm_shm_map(const char *name, na_size_t length, na_bool_t create) { na_size_t page_size = (na_size_t) hg_mem_get_page_size(); - void *ret = NULL; /* Check alignment */ - NA_CHECK_WARNING(buf_size / page_size * page_size != buf_size, - "Not aligned properly, page size=%zu bytes, buf size=%zu bytes", - page_size, buf_size); + NA_CHECK_WARNING(length / page_size * page_size != length, + "Not aligned properly, page size=%zu bytes, length=%zu bytes", + page_size, length); - ret = hg_mem_shm_map(name, buf_size, create); - - return ret; + return hg_mem_shm_map(name, length, create); } /*---------------------------------------------------------------------------*/ static na_return_t -na_sm_close_shared_buf(const char *filename, void *buf, size_t buf_size) +na_sm_shm_unmap(const char *name, void *addr, na_size_t length) { - return hg_mem_shm_unmap(filename, buf, buf_size); + return (hg_mem_shm_unmap(name, addr, length) == HG_UTIL_SUCCESS) + ? NA_SUCCESS + : na_sm_errno_to_na(errno); } /*---------------------------------------------------------------------------*/ -static na_return_t -na_sm_create_sock(const char *pathname, na_bool_t na_listen, int *sock) +static int +na_sm_shm_cleanup(const char *fpath, const struct stat NA_UNUSED *sb, + int NA_UNUSED typeflag, struct FTW NA_UNUSED *ftwbuf) { - struct sockaddr_un addr; - char *dup_path = NULL; - na_return_t ret = NA_SUCCESS; - int fd = -1, rc; - - /* Create a non-blocking socket so that we can poll for incoming connections */ -#ifdef SOCK_NONBLOCK - fd = socket(AF_UNIX, SOCK_SEQPACKET | SOCK_NONBLOCK, 0); -#else - fd = socket(AF_UNIX, SOCK_STREAM, 0); -#endif - NA_CHECK_ERROR(fd == -1, error, ret, NA_PROTOCOL_ERROR, - "socket() failed (%s)", strerror(errno)); + const char *prefix = NA_SM_SHM_PATH "/" NA_SM_SHM_PREFIX "_"; + int ret = 0; -#ifndef SOCK_NONBLOCK - rc = fcntl(fd, F_SETFL, O_NONBLOCK); - NA_CHECK_ERROR(rc == -1, error, ret, NA_PROTOCOL_ERROR, - "fcntl() failed (%s)", strerror(errno)); -#endif + if (strncmp(fpath, prefix, strlen(prefix)) == 0) { + const char *shm_name = fpath + strlen(NA_SM_SHM_PATH "/"); + char *username = getlogin_safe(); - memset(&addr, 0, sizeof(struct sockaddr_un)); - addr.sun_family = AF_UNIX; - NA_CHECK_ERROR(strlen(pathname) + strlen("/sock") > sizeof(addr.sun_path) - 1, - error, ret, NA_OVERFLOW, "Exceeds maximum AF UNIX socket path length"); - strcpy(addr.sun_path, pathname); - strcat(addr.sun_path, "/sock"); - - if (na_listen) { - char stat_path[NA_SM_MAX_FILENAME]; - char *path_ptr; - - dup_path = strdup(pathname); - NA_CHECK_ERROR(dup_path == NULL, error, ret, NA_NOMEM, - "Could not dup pathname"); - path_ptr = dup_path; - - memset(stat_path, '\0', NA_SM_MAX_FILENAME); - if (dup_path[0] == '/') { - path_ptr++; - stat_path[0] = '/'; + if (strncmp(shm_name + strlen(NA_SM_SHM_PREFIX "_"), username, + strlen(username)) == 0) { + NA_LOG_DEBUG("shm_unmap() %s", shm_name); + ret = hg_mem_shm_unmap(shm_name, NULL, 0); } + } - /* Create path */ - while (path_ptr) { - struct stat sb; - char *current = strtok_r(path_ptr, "/", &path_ptr); - if (!current) - break; - - strcat(stat_path, current); - if (stat(stat_path, &sb) == -1) { - rc = mkdir(stat_path, 0775); - NA_CHECK_ERROR(rc == -1 && errno != EEXIST, error, ret, - NA_PROTOCOL_ERROR, "Could not create directory: %s (%s)", - stat_path, strerror(errno)); - } - strcat(stat_path, "/"); - } + return ret; +} - /* Bind */ - rc = bind(fd, (struct sockaddr *) &addr, sizeof(struct sockaddr_un)); - NA_CHECK_ERROR(rc == -1, error, ret, NA_PROTOCOL_ERROR, - "bind() failed (%s)", strerror(errno)); +/*---------------------------------------------------------------------------*/ +static void +na_sm_msg_queue_init(struct na_sm_msg_queue *na_sm_queue) +{ + struct hg_atomic_queue *hg_atomic_queue = + (struct hg_atomic_queue *) na_sm_queue; + unsigned int count = NA_SM_NUM_BUFS; - /* Listen */ - rc = listen(fd, NA_SM_LISTEN_BACKLOG); - NA_CHECK_ERROR(rc == -1, error, ret, NA_PROTOCOL_ERROR, - "listen() failed (%s)", strerror(errno)); - } else { - /* Connect */ - rc = connect(fd, (struct sockaddr *) &addr, sizeof(struct sockaddr_un)); - NA_CHECK_ERROR(rc == -1, error, ret, NA_PROTOCOL_ERROR, - "connect() failed (%s)", strerror(errno)); + hg_atomic_queue->prod_size = hg_atomic_queue->cons_size = count; + hg_atomic_queue->prod_mask = hg_atomic_queue->cons_mask = count - 1; + hg_atomic_init32(&hg_atomic_queue->prod_head, 0); + hg_atomic_init32(&hg_atomic_queue->cons_head, 0); + hg_atomic_init32(&hg_atomic_queue->prod_tail, 0); + hg_atomic_init32(&hg_atomic_queue->cons_tail, 0); +} + +/*---------------------------------------------------------------------------*/ +static NA_INLINE na_bool_t +na_sm_msg_queue_push( + struct na_sm_msg_queue *na_sm_queue, na_sm_msg_hdr_t msg_hdr) +{ + int rc = hg_atomic_queue_push( + (struct hg_atomic_queue *) na_sm_queue, (void *) msg_hdr.val); + + return (likely(rc == HG_UTIL_SUCCESS)) ? NA_TRUE : NA_FALSE; +} + +/*---------------------------------------------------------------------------*/ +static NA_INLINE na_bool_t +na_sm_msg_queue_pop( + struct na_sm_msg_queue *na_sm_queue, na_sm_msg_hdr_t *msg_hdr_ptr) +{ + msg_hdr_ptr->val = (na_uint64_t) hg_atomic_queue_pop_mc( + (struct hg_atomic_queue *) na_sm_queue); + + return (likely(msg_hdr_ptr->val)) ? NA_TRUE : NA_FALSE; +} + +/*---------------------------------------------------------------------------*/ +static NA_INLINE na_bool_t +na_sm_msg_queue_is_empty(struct na_sm_msg_queue *na_sm_queue) +{ + return hg_atomic_queue_is_empty((struct hg_atomic_queue *) na_sm_queue); +} + +/*---------------------------------------------------------------------------*/ +static void +na_sm_cmd_queue_init(struct na_sm_cmd_queue *na_sm_queue) +{ + struct hg_atomic_queue *hg_atomic_queue = + (struct hg_atomic_queue *) na_sm_queue; + unsigned int count = NA_SM_MAX_PEERS * 2; + + hg_atomic_queue->prod_size = hg_atomic_queue->cons_size = count; + hg_atomic_queue->prod_mask = hg_atomic_queue->cons_mask = count - 1; + hg_atomic_init32(&hg_atomic_queue->prod_head, 0); + hg_atomic_init32(&hg_atomic_queue->cons_head, 0); + hg_atomic_init32(&hg_atomic_queue->prod_tail, 0); + hg_atomic_init32(&hg_atomic_queue->cons_tail, 0); +} + +/*---------------------------------------------------------------------------*/ +static NA_INLINE na_bool_t +na_sm_cmd_queue_push( + struct na_sm_cmd_queue *na_sm_queue, na_sm_cmd_hdr_t cmd_hdr) +{ + int rc = hg_atomic_queue_push( + (struct hg_atomic_queue *) na_sm_queue, (void *) cmd_hdr.val); + + return (likely(rc == HG_UTIL_SUCCESS)) ? NA_TRUE : NA_FALSE; +} + +/*---------------------------------------------------------------------------*/ +static NA_INLINE na_bool_t +na_sm_cmd_queue_pop( + struct na_sm_cmd_queue *na_sm_queue, na_sm_cmd_hdr_t *cmd_hdr_ptr) +{ + cmd_hdr_ptr->val = (na_uint64_t) hg_atomic_queue_pop_mc( + (struct hg_atomic_queue *) na_sm_queue); + + return (likely(cmd_hdr_ptr->val)) ? NA_TRUE : NA_FALSE; +} + +/*---------------------------------------------------------------------------*/ +static NA_INLINE na_bool_t +na_sm_cmd_queue_is_empty(struct na_sm_cmd_queue *na_sm_queue) +{ + return hg_atomic_queue_is_empty((struct hg_atomic_queue *) na_sm_queue); +} + +/*---------------------------------------------------------------------------*/ +static NA_INLINE na_uint64_t +na_sm_addr_to_key(pid_t pid, na_uint8_t id) +{ + return (((na_uint64_t) pid) << 32 | id); +} + +/*---------------------------------------------------------------------------*/ +static NA_INLINE unsigned int +na_sm_addr_key_hash(hg_hash_table_key_t vlocation) +{ + /* Hashing through PIDs should be sufficient in practice */ + return (unsigned int) (*((na_uint64_t *) vlocation) >> 32); +} + +/*---------------------------------------------------------------------------*/ +static NA_INLINE int +na_sm_addr_key_equal( + hg_hash_table_key_t vlocation1, hg_hash_table_key_t vlocation2) +{ + return *((na_uint64_t *) vlocation1) == *((na_uint64_t *) vlocation2); +} + +/*---------------------------------------------------------------------------*/ +static na_return_t +na_sm_string_to_addr(const char *str, pid_t *pid, na_uint8_t *id) +{ + char *name = NULL, *short_name = NULL; + na_return_t ret = NA_SUCCESS; + + /** + * Clean up name, strings can be of the format: + * :// + */ + name = strdup(str); + NA_CHECK_ERROR( + name == NULL, done, ret, NA_NOMEM, "Could not duplicate string"); + + if (strstr(name, ":") != NULL) { + strtok_r(name, ":", &short_name); + short_name += 2; + } else + short_name = name; + + /* Get PID / ID from name */ + sscanf(short_name, "%d/%" SCNu8, pid, id); + +done: + free(name); + + return ret; +} + +/*---------------------------------------------------------------------------*/ +static na_return_t +na_sm_region_open(const char *username, pid_t pid, na_uint8_t id, + na_bool_t create, struct na_sm_region **region) +{ + char shm_name[NA_SM_MAX_FILENAME] = {'\0'}; + struct na_sm_region *na_sm_region = NULL; + na_return_t ret = NA_SUCCESS; + int rc; + + /* Generate SHM object name */ + rc = NA_SM_GEN_SHM_NAME( + shm_name, NA_SM_MAX_FILENAME, username, (int) pid, id); + NA_CHECK_ERROR(rc < 0 || rc > NA_SM_MAX_FILENAME, done, ret, NA_OVERFLOW, + "NA_SM_GEN_SHM_NAME() failed, rc: %d", rc); + + /* Open SHM object */ + NA_LOG_DEBUG("shm_map() %s", shm_name); + na_sm_region = (struct na_sm_region *) na_sm_shm_map( + shm_name, sizeof(struct na_sm_region), create); + NA_CHECK_ERROR(na_sm_region == NULL, done, ret, NA_NODEV, + "Could not map new SM region (%s)", shm_name); + + if (create) { + int i; + + /* Initialize copy buf (all buffers are available by default) */ + hg_atomic_init64( + &na_sm_region->copy_bufs.available.val, ~((hg_util_int64_t) 0)); + memset(&na_sm_region->copy_bufs.buf, 0, + sizeof(na_sm_region->copy_bufs.buf)); + + /* Initialize locks */ + for (i = 0; i < NA_SM_NUM_BUFS; i++) + hg_thread_spin_init(&na_sm_region->copy_bufs.buf_locks[i]); + + /* Initialize queue pairs */ + for (i = 0; i < 4; i++) + hg_atomic_init64( + &na_sm_region->available.val[i], ~((hg_util_int64_t) 0)); + + for (i = 0; i < NA_SM_MAX_PEERS; i++) { + na_sm_msg_queue_init(&na_sm_region->queue_pairs[i].rx_queue); + na_sm_msg_queue_init(&na_sm_region->queue_pairs[i].tx_queue); + } + + /* Initialize command queue */ + na_sm_cmd_queue_init(&na_sm_region->cmd_queue); + } + + *region = na_sm_region; + +done: + return ret; +} + +/*---------------------------------------------------------------------------*/ +static na_return_t +na_sm_region_close(const char *username, pid_t pid, na_uint8_t id, + na_bool_t remove, struct na_sm_region *region) +{ + char shm_name[NA_SM_MAX_FILENAME] = {'\0'}; + const char *shm_name_ptr = NULL; + na_return_t ret = NA_SUCCESS; + + if (remove) { + int rc = NA_SM_GEN_SHM_NAME( + shm_name, NA_SM_MAX_FILENAME, username, (int) pid, id); + NA_CHECK_ERROR(rc < 0 || rc > NA_SM_MAX_FILENAME, done, ret, + NA_OVERFLOW, "NA_SM_GEN_SHM_NAME() failed, rc: %d", rc); + shm_name_ptr = shm_name; + } + + NA_LOG_DEBUG("shm_unmap() %s", shm_name_ptr); + ret = na_sm_shm_unmap(shm_name_ptr, region, sizeof(struct na_sm_region)); + NA_CHECK_NA_ERROR( + done, ret, "Could not unmap SM region (%s)", shm_name_ptr); + +done: + return ret; +} + +/*---------------------------------------------------------------------------*/ +static na_return_t +na_sm_sock_open( + const char *username, pid_t pid, na_uint8_t id, na_bool_t create, int *sock) +{ + int socket_type = SOCK_DGRAM, /* reliable with AF_UNIX */ + fd = -1, rc; + char pathname[NA_SM_MAX_FILENAME] = {'\0'}; + na_bool_t created_sock_path = NA_FALSE; + na_return_t ret = NA_SUCCESS; + + /* Create a non-blocking socket so that we can poll for incoming connections + */ +#ifdef SOCK_NONBLOCK + socket_type |= SOCK_NONBLOCK; +#endif + fd = socket(AF_UNIX, socket_type, 0); + NA_CHECK_ERROR(fd == -1, error, ret, na_sm_errno_to_na(errno), + "socket() failed (%s)", strerror(errno)); + +#ifndef SOCK_NONBLOCK + rc = fcntl(fd, F_SETFL, O_NONBLOCK); + NA_CHECK_ERROR(rc == -1, error, ret, na_sm_errno_to_na(errno), + "fcntl() failed (%s)", strerror(errno)); +#endif + + if (create) { + struct sockaddr_un addr; + + /* Generate named socket path */ + rc = NA_SM_GEN_SOCK_PATH( + pathname, NA_SM_MAX_FILENAME, username, pid, id); + NA_CHECK_ERROR(rc < 0 || rc > NA_SM_MAX_FILENAME, error, ret, + NA_OVERFLOW, "NA_SM_GEN_SOCK_PATH() failed, rc: %d", rc); + + memset(&addr, 0, sizeof(struct sockaddr_un)); + addr.sun_family = AF_UNIX; + NA_CHECK_ERROR(strlen(pathname) + strlen(NA_SM_SOCK_NAME) > + sizeof(addr.sun_path) - 1, + error, ret, NA_OVERFLOW, + "Exceeds maximum AF UNIX socket path length"); + strcpy(addr.sun_path, pathname); + strcat(addr.sun_path, NA_SM_SOCK_NAME); + + /* Create path */ + ret = na_sm_sock_path_create(pathname); + NA_CHECK_NA_ERROR( + error, ret, "Could not create socket path (%s)", pathname); + created_sock_path = NA_TRUE; + + /* Bind and create named socket */ + NA_LOG_DEBUG("bind() %s", addr.sun_path); + rc = bind( + fd, (const struct sockaddr *) &addr, (socklen_t) SUN_LEN(&addr)); + NA_CHECK_ERROR(rc == -1, error, ret, na_sm_errno_to_na(errno), + "bind() failed (%s)", strerror(errno)); } *sock = fd; - free(dup_path); return ret; error: @@ -1117,42 +1368,48 @@ na_sm_create_sock(const char *pathname, na_bool_t na_listen, int *sock) rc = close(fd); NA_CHECK_ERROR_DONE(rc == -1, "close() failed (%s)", strerror(errno)); } - free(dup_path); + if (created_sock_path) { + na_return_t err_ret = na_sm_sock_path_remove(pathname); + NA_CHECK_ERROR_DONE(err_ret != NA_SUCCESS, + "na_sm_remove_sock_path() failed (%s)", pathname); + } return ret; } /*---------------------------------------------------------------------------*/ static na_return_t -na_sm_close_sock(int sock, const char *pathname) +na_sm_sock_close( + const char *username, pid_t pid, na_uint8_t id, na_bool_t remove, int sock) { na_return_t ret = NA_SUCCESS; int rc; + NA_LOG_DEBUG("Closing sock %d", sock); rc = close(sock); - NA_CHECK_ERROR(rc == -1, done, ret, NA_PROTOCOL_ERROR, + NA_CHECK_ERROR(rc == -1, done, ret, na_sm_errno_to_na(errno), "close() failed (%s)", strerror(errno)); - if (pathname) { - char dup_path[NA_SM_MAX_FILENAME]; - char *path_ptr = NULL; + if (remove) { + char pathname[NA_SM_MAX_FILENAME] = {'\0'}; + struct sockaddr_un addr; + + /* Generate named socket path */ + rc = NA_SM_GEN_SOCK_PATH( + pathname, NA_SM_MAX_FILENAME, username, pid, id); + NA_CHECK_ERROR(rc < 0 || rc > NA_SM_MAX_FILENAME, done, ret, + NA_OVERFLOW, "NA_SM_GEN_SOCK_PATH() failed, rc: %d", rc); - strcpy(dup_path, pathname); - strcat(dup_path, "/sock"); + strcpy(addr.sun_path, pathname); + strcat(addr.sun_path, NA_SM_SOCK_NAME); - rc = unlink(dup_path); - NA_CHECK_ERROR(rc == -1, done, ret, NA_PROTOCOL_ERROR, + NA_LOG_DEBUG("unlink() %s", addr.sun_path); + rc = unlink(addr.sun_path); + NA_CHECK_ERROR(rc == -1, done, ret, na_sm_errno_to_na(errno), "unlink() failed (%s)", strerror(errno)); - /* Delete path */ - path_ptr = strrchr(dup_path, '/'); - while (path_ptr) { - *path_ptr = '\0'; - if (rmdir(dup_path) == -1) { - /* Silently ignore */ - } - path_ptr = strrchr(dup_path, '/'); - } + ret = na_sm_sock_path_remove(pathname); + NA_CHECK_NA_ERROR(done, ret, "Could not remove %s path", pathname); } done: @@ -1160,282 +1417,818 @@ na_sm_close_sock(int sock, const char *pathname) } /*---------------------------------------------------------------------------*/ -static int -na_sm_cleanup_file(const char *fpath, const struct stat NA_UNUSED *sb, - int NA_UNUSED typeflag, struct FTW NA_UNUSED *ftwbuf) +static na_return_t +na_sm_sock_path_create(const char *pathname) { - return remove(fpath); + char *dup_path = NULL, *path_ptr; + char stat_path[NA_SM_MAX_FILENAME] = {'\0'}; + na_return_t ret = NA_SUCCESS; + + dup_path = strdup(pathname); + NA_CHECK_ERROR( + dup_path == NULL, done, ret, NA_NOMEM, "Could not dup pathname"); + path_ptr = dup_path; + + /* Skip leading '/' */ + if (dup_path[0] == '/') { + path_ptr++; + stat_path[0] = '/'; + } + + /* Create path */ + while (path_ptr != NULL) { + char *current = strtok_r(path_ptr, "/", &path_ptr); + struct stat sb; + + if (!current) + break; + + strcat(stat_path, current); + if (stat(stat_path, &sb) == -1) { + int rc; + NA_LOG_DEBUG("mkdir %s", stat_path); + rc = mkdir(stat_path, 0775); + NA_CHECK_ERROR(rc == -1 && errno != EEXIST, done, ret, + na_sm_errno_to_na(errno), "Could not create directory: %s (%s)", + stat_path, strerror(errno)); + } + strcat(stat_path, "/"); + } + +done: + free(dup_path); + + return ret; } /*---------------------------------------------------------------------------*/ -static int -na_sm_cleanup_shm(const char *fpath, const struct stat NA_UNUSED *sb, - int NA_UNUSED typeflag, struct FTW NA_UNUSED *ftwbuf) +static na_return_t +na_sm_sock_path_remove(const char *pathname) { - const char *prefix = NA_SM_SHM_PATH "/" NA_SM_SHM_PREFIX "_"; - int ret = 0; + char dup_path[NA_SM_MAX_FILENAME] = {'\0'}; + char *path_ptr = NULL; + na_return_t ret = NA_SUCCESS; - if (strncmp(fpath, prefix, strlen(prefix)) == 0) { - const char *file = fpath + strlen(NA_SM_SHM_PATH "/"); - char *username = getlogin_safe(); + strcpy(dup_path, pathname); - if (strncmp(file + strlen(NA_SM_SHM_PREFIX "_"), - username, strlen(username)) == 0) - ret = hg_mem_shm_unmap(file, NULL, 0); + /* Delete path */ + path_ptr = strrchr(dup_path, '/'); + while (path_ptr) { + *path_ptr = '\0'; + NA_LOG_DEBUG("rmdir %s", dup_path); + if (rmdir(dup_path) == -1) { + /* Silently ignore */ + } + path_ptr = strrchr(dup_path, '/'); } return ret; } /*---------------------------------------------------------------------------*/ -#ifndef HG_UTIL_HAS_SYSEVENTFD_H - static int -na_sm_event_create(const char *filename) +na_sm_sock_path_cleanup(const char *fpath, const struct stat NA_UNUSED *sb, + int NA_UNUSED typeflag, struct FTW NA_UNUSED *ftwbuf) { + return remove(fpath); +} + +/*---------------------------------------------------------------------------*/ +static na_return_t +na_sm_event_create(const char NA_UNUSED *username, pid_t NA_UNUSED pid, + na_uint8_t NA_UNUSED id, na_uint8_t NA_UNUSED pair_index, + unsigned char NA_UNUSED pair, int *event) +{ + na_return_t ret = NA_SUCCESS; int fd = -1; + +#ifdef HG_UTIL_HAS_SYSEVENTFD_H + fd = hg_event_create(); + NA_CHECK_ERROR(fd == -1, error, ret, na_sm_errno_to_na(errno), + "hg_event_create() failed"); +#else + char fifo_name[NA_SM_MAX_FILENAME] = {'\0'}; int rc; + /** + * If eventfd is not supported, we need to explicitly use named pipes in + * this case as kqueue file descriptors cannot be exchanged through + * ancillary data. + */ + rc = NA_SM_GEN_FIFO_NAME( + fifo_name, NA_SM_MAX_FILENAME, username, pid, id, pair_index, pair); + NA_CHECK_ERROR(rc < 0 || rc > NA_SM_MAX_FILENAME, error, ret, NA_OVERFLOW, + "NA_SM_GEN_FIFO_NAME() failed, rc: %d", rc); + /* Create FIFO */ - rc = mkfifo(filename, S_IRUSR | S_IWUSR); - NA_CHECK_ERROR_NORET(rc == -1, error, "mkfifo() failed (%s)", - strerror(errno)); + NA_LOG_DEBUG("mkfifo() %s", fifo_name); + rc = mkfifo(fifo_name, S_IRUSR | S_IWUSR); + NA_CHECK_ERROR(rc == -1, error, ret, na_sm_errno_to_na(errno), + "mkfifo() failed (%s)", strerror(errno)); /* Open FIFO (RDWR for convenience) */ - fd = open(filename, O_RDWR); - NA_CHECK_ERROR_NORET(fd == -1, error, "open() failed (%s)", strerror(errno)); + fd = open(fifo_name, O_RDWR); + NA_CHECK_ERROR(fd == -1, error, ret, na_sm_errno_to_na(errno), + "open() failed (%s)", strerror(errno)); /* Set FIFO to be non-blocking */ rc = fcntl(fd, F_SETFL, O_NONBLOCK); - NA_CHECK_ERROR_NORET(rc == -1, error, "fcntl() failed (%s)", - strerror(errno)); + NA_CHECK_ERROR(rc == -1, error, ret, na_sm_errno_to_na(errno), + "fcntl() failed (%s)", strerror(errno)); +#endif + NA_LOG_DEBUG("Created event %d", fd); - return fd; + *event = fd; + + return ret; error: +#ifndef HG_UTIL_HAS_SYSEVENTFD_H if (fd != -1) { rc = close(fd); NA_CHECK_ERROR_DONE(rc == -1, "close() failed (%s)", strerror(errno)); } +#endif - return -1; + return ret; } /*---------------------------------------------------------------------------*/ -na_return_t -na_sm_event_destroy(const char *filename, int fd) +static na_return_t +na_sm_event_destroy(const char NA_UNUSED *username, pid_t NA_UNUSED pid, + na_uint8_t NA_UNUSED id, na_uint8_t NA_UNUSED pair_index, + unsigned char NA_UNUSED pair, na_bool_t NA_UNUSED remove, int event) { na_return_t ret = NA_SUCCESS; int rc; - rc = close(fd); - NA_CHECK_ERROR(rc == -1, done, ret, NA_PROTOCOL_ERROR, + NA_LOG_DEBUG("Closing event %d", event); +#ifdef HG_UTIL_HAS_SYSEVENTFD_H + rc = hg_event_destroy(event); + NA_CHECK_ERROR(rc == HG_UTIL_FAIL, done, ret, na_sm_errno_to_na(errno), + "hg_event_destroy() failed"); +#else + rc = close(event); + NA_CHECK_ERROR(rc == -1, done, ret, na_sm_errno_to_na(errno), "close() failed (%s)", strerror(errno)); - if (filename) { - rc = unlink(filename); - NA_CHECK_ERROR(rc == -1, done, ret, NA_PROTOCOL_ERROR, + if (remove) { + char fifo_name[NA_SM_MAX_FILENAME] = {'\0'}; + + rc = NA_SM_GEN_FIFO_NAME( + fifo_name, NA_SM_MAX_FILENAME, username, pid, id, pair_index, pair); + NA_CHECK_ERROR(rc < 0 || rc > NA_SM_MAX_FILENAME, done, ret, + NA_OVERFLOW, "NA_SM_GEN_FIFO_NAME() failed, rc: %d", rc); + + NA_LOG_DEBUG("unlink() %s", fifo_name); + rc = unlink(fifo_name); + NA_CHECK_ERROR(rc == -1, done, ret, na_sm_errno_to_na(errno), "unlink() failed (%s)", strerror(errno)); } +#endif done: return ret; } /*---------------------------------------------------------------------------*/ -na_return_t -na_sm_event_set(int fd) +static NA_INLINE na_return_t +na_sm_event_set(int event) { na_return_t ret = NA_SUCCESS; +#ifdef HG_UTIL_HAS_SYSEVENTFD_H + int rc; + + rc = hg_event_set(event); + NA_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, na_sm_errno_to_na(errno), + "hg_event_set() failed"); +#else uint64_t count = 1; ssize_t s; - s = write(fd, &count, sizeof(uint64_t)); - NA_CHECK_ERROR(s != sizeof(uint64_t), done, ret, NA_PROTOCOL_ERROR, + s = write(event, &count, sizeof(uint64_t)); + NA_CHECK_ERROR(s != sizeof(uint64_t), done, ret, na_sm_errno_to_na(errno), "write() failed (%s)", strerror(errno)); +#endif done: return ret; } /*---------------------------------------------------------------------------*/ -na_return_t -na_sm_event_get(int fd, na_bool_t *signaled) +static NA_INLINE na_return_t +na_sm_event_get(int event, na_bool_t *signaled) { na_return_t ret = NA_SUCCESS; +#ifdef HG_UTIL_HAS_SYSEVENTFD_H + int rc; + + rc = hg_event_get(event, (hg_util_bool_t *) signaled); + NA_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, na_sm_errno_to_na(errno), + "hg_event_get() failed"); +#else uint64_t count = 1; ssize_t s; - s = read(fd, &count, sizeof(uint64_t)); + s = read(event, &count, sizeof(uint64_t)); if (s != sizeof(uint64_t)) { if (likely(errno == EAGAIN)) { *signaled = NA_FALSE; goto done; } else - NA_GOTO_ERROR(done, ret, NA_PROTOCOL_ERROR, "read() failed (%s)", - strerror(errno)); + NA_GOTO_ERROR(done, ret, na_sm_errno_to_na(errno), + "read() failed (%s)", strerror(errno)); } *signaled = NA_TRUE; +#endif + +done: + return ret; +} + +/*---------------------------------------------------------------------------*/ +static na_return_t +na_sm_poll_register( + hg_poll_set_t *poll_set, int fd, hg_poll_cb_t poll_cb, void *poll_arg) +{ + unsigned int flags = HG_POLLIN; + na_return_t ret = NA_SUCCESS; + int rc; + + rc = hg_poll_add(poll_set, fd, flags, poll_cb, poll_arg); + NA_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, na_sm_errno_to_na(errno), + "hg_poll_add() failed"); + +done: + return ret; +} + +/*---------------------------------------------------------------------------*/ +static na_return_t +na_sm_poll_deregister(hg_poll_set_t *poll_set, int fd) +{ + na_return_t ret = NA_SUCCESS; + int rc; + + rc = hg_poll_remove(poll_set, fd); + NA_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, na_sm_errno_to_na(errno), + "hg_poll_remove() failed"); + +done: + return ret; +} + +/*---------------------------------------------------------------------------*/ +static na_return_t +na_sm_endpoint_open(struct na_sm_endpoint *na_sm_endpoint, const char *username, + pid_t pid, na_uint8_t id, na_bool_t listen, na_bool_t no_wait) +{ + struct na_sm_region *shared_region = NULL; + na_uint8_t queue_pair_idx = 0; + na_bool_t queue_pair_reserved = NA_FALSE, sock_registered = NA_FALSE; + int tx_notify = -1; + na_return_t ret = NA_SUCCESS, err_ret; + + /* Save listen state */ + na_sm_endpoint->listen = listen; + + /* Initialize queues */ + HG_QUEUE_INIT(&na_sm_endpoint->unexpected_msg_queue.queue); + hg_thread_spin_init(&na_sm_endpoint->unexpected_msg_queue.lock); + + HG_QUEUE_INIT(&na_sm_endpoint->unexpected_op_queue.queue); + hg_thread_spin_init(&na_sm_endpoint->unexpected_op_queue.lock); + + HG_QUEUE_INIT(&na_sm_endpoint->expected_op_queue.queue); + hg_thread_spin_init(&na_sm_endpoint->expected_op_queue.lock); + + HG_QUEUE_INIT(&na_sm_endpoint->retry_op_queue.queue); + hg_thread_spin_init(&na_sm_endpoint->retry_op_queue.lock); + + /* Initialize poll addr list */ + HG_LIST_INIT(&na_sm_endpoint->poll_addr_list.list); + hg_thread_spin_init(&na_sm_endpoint->poll_addr_list.lock); + + /* Create addr hash-table */ + na_sm_endpoint->addr_map.map = + hg_hash_table_new(na_sm_addr_key_hash, na_sm_addr_key_equal); + NA_CHECK_ERROR(na_sm_endpoint->addr_map.map == NULL, error, ret, NA_NOMEM, + "hg_hash_table_new() failed"); + hg_hash_table_register_free_functions( + na_sm_endpoint->addr_map.map, free, free); + hg_thread_rwlock_init(&na_sm_endpoint->addr_map.lock); + + if (listen) { + /* If we're listening, create a new shm region */ + ret = na_sm_region_open(username, pid, id, NA_TRUE, &shared_region); + NA_CHECK_NA_ERROR(error, ret, "Could not open shared-memory region"); + + /* Reserve queue pair for loopback */ + ret = na_sm_queue_pair_reserve(shared_region, &queue_pair_idx); + NA_CHECK_NA_ERROR(error, ret, "Could not reserve queue pair"); + queue_pair_reserved = NA_TRUE; + } + + if (!no_wait) { + /* Create poll set to wait for events */ + na_sm_endpoint->poll_set = hg_poll_create(); + NA_CHECK_ERROR(na_sm_endpoint->poll_set == NULL, error, ret, + na_sm_errno_to_na(errno), "Cannot create poll set"); + + /* Create endpoint sock */ + ret = na_sm_sock_open(username, pid, id, listen, &na_sm_endpoint->sock); + NA_CHECK_NA_ERROR(error, ret, "Could not open sock"); + + if (listen) { + NA_LOG_DEBUG( + "Registering sock %d for polling", na_sm_endpoint->sock); + /* Add sock to poll set (ony required if we're listening) */ + ret = na_sm_poll_register(na_sm_endpoint->poll_set, + na_sm_endpoint->sock, na_sm_progress_sock, na_sm_endpoint); + NA_CHECK_NA_ERROR(error, ret, "Could not add sock to poll set"); + sock_registered = NA_TRUE; + } + + /* Create local tx signaling event */ + tx_notify = hg_event_create(); + NA_CHECK_ERROR(tx_notify == -1, error, ret, na_sm_errno_to_na(errno), + "hg_event_create() failed"); + } else { + na_sm_endpoint->sock = -1; + } + + /* Allocate source address */ + ret = na_sm_addr_create(na_sm_endpoint, shared_region, pid, id, + queue_pair_idx, tx_notify, -1, NA_FALSE, &na_sm_endpoint->source_addr); + NA_CHECK_NA_ERROR(error, ret, "Could not allocate source address"); + + /* Add source tx notify to poll set for local notifications */ + if (!no_wait) { + NA_LOG_DEBUG("Registering tx notify %d for polling", tx_notify); + ret = na_sm_poll_register(na_sm_endpoint->poll_set, tx_notify, + na_sm_progress_tx_notify, na_sm_endpoint->source_addr); + NA_CHECK_NA_ERROR(error, ret, "Could not add tx notify to poll set"); + } + + return ret; + +error: + if (na_sm_endpoint->source_addr) + free(na_sm_endpoint->source_addr); + if (tx_notify > 0) + hg_event_destroy(tx_notify); + if (sock_registered) { + err_ret = na_sm_poll_deregister( + na_sm_endpoint->poll_set, na_sm_endpoint->sock); + NA_CHECK_ERROR_DONE( + err_ret != NA_SUCCESS, "na_sm_poll_deregister() failed"); + } + if (na_sm_endpoint->sock > 0) { + err_ret = + na_sm_sock_close(username, pid, id, listen, na_sm_endpoint->sock); + NA_CHECK_ERROR_DONE(err_ret != NA_SUCCESS, "na_sm_sock_close() failed"); + } + if (na_sm_endpoint->poll_set) + hg_poll_destroy(na_sm_endpoint->poll_set); + if (queue_pair_reserved) + na_sm_queue_pair_release(shared_region, queue_pair_idx); + if (shared_region) + na_sm_region_close(username, pid, id, NA_TRUE, shared_region); + if (na_sm_endpoint->addr_map.map) { + hg_hash_table_free(na_sm_endpoint->addr_map.map); + hg_thread_rwlock_destroy(&na_sm_endpoint->addr_map.lock); + } + + hg_thread_spin_destroy(&na_sm_endpoint->unexpected_msg_queue.lock); + hg_thread_spin_destroy(&na_sm_endpoint->unexpected_op_queue.lock); + hg_thread_spin_destroy(&na_sm_endpoint->expected_op_queue.lock); + hg_thread_spin_destroy(&na_sm_endpoint->retry_op_queue.lock); + hg_thread_spin_destroy(&na_sm_endpoint->poll_addr_list.lock); + + return ret; +} + +/*---------------------------------------------------------------------------*/ +static na_return_t +na_sm_endpoint_close( + struct na_sm_endpoint *na_sm_endpoint, const char *username) +{ + struct na_sm_addr *source_addr = na_sm_endpoint->source_addr; + na_return_t ret = NA_SUCCESS; + na_bool_t empty; + + /* Check that poll addr list is empty */ + hg_thread_spin_lock(&na_sm_endpoint->poll_addr_list.lock); + empty = HG_LIST_IS_EMPTY(&na_sm_endpoint->poll_addr_list.list); + hg_thread_spin_unlock(&na_sm_endpoint->poll_addr_list.lock); + + if (!empty) { + struct na_sm_addr *na_sm_addr; + + hg_thread_spin_lock(&na_sm_endpoint->poll_addr_list.lock); + na_sm_addr = HG_LIST_FIRST(&na_sm_endpoint->poll_addr_list.list); + while (na_sm_addr) { + struct na_sm_addr *next = HG_LIST_NEXT(na_sm_addr, entry); + HG_LIST_REMOVE(na_sm_addr, entry); + + /* Destroy remaining addresses */ + ret = na_sm_addr_destroy(na_sm_endpoint, username, na_sm_addr); + NA_CHECK_NA_ERROR(done, ret, "Could not remove address"); + + na_sm_addr = next; + } + /* Sanity check */ + empty = HG_LIST_IS_EMPTY(&na_sm_endpoint->poll_addr_list.list); + hg_thread_spin_unlock(&na_sm_endpoint->poll_addr_list.lock); + } + NA_CHECK_ERROR(empty == NA_FALSE, done, ret, NA_BUSY, + "Poll addr list should be empty"); + + /* Check that unexpected message queue is empty */ + hg_thread_spin_lock(&na_sm_endpoint->unexpected_msg_queue.lock); + empty = HG_QUEUE_IS_EMPTY(&na_sm_endpoint->unexpected_msg_queue.queue); + hg_thread_spin_unlock(&na_sm_endpoint->unexpected_msg_queue.lock); + NA_CHECK_ERROR(empty == NA_FALSE, done, ret, NA_BUSY, + "Unexpected msg queue should be empty"); + + /* Check that unexpected op queue is empty */ + hg_thread_spin_lock(&na_sm_endpoint->unexpected_op_queue.lock); + empty = HG_QUEUE_IS_EMPTY(&na_sm_endpoint->unexpected_op_queue.queue); + hg_thread_spin_unlock(&na_sm_endpoint->unexpected_op_queue.lock); + NA_CHECK_ERROR(empty == NA_FALSE, done, ret, NA_BUSY, + "Unexpected op queue should be empty"); + + /* Check that expected op queue is empty */ + hg_thread_spin_lock(&na_sm_endpoint->expected_op_queue.lock); + empty = HG_QUEUE_IS_EMPTY(&na_sm_endpoint->expected_op_queue.queue); + hg_thread_spin_unlock(&na_sm_endpoint->expected_op_queue.lock); + NA_CHECK_ERROR(empty == NA_FALSE, done, ret, NA_BUSY, + "Expected op queue should be empty"); + + /* Check that retry op queue is empty */ + hg_thread_spin_lock(&na_sm_endpoint->retry_op_queue.lock); + empty = HG_QUEUE_IS_EMPTY(&na_sm_endpoint->retry_op_queue.queue); + hg_thread_spin_unlock(&na_sm_endpoint->retry_op_queue.lock); + NA_CHECK_ERROR(empty == NA_FALSE, done, ret, NA_BUSY, + "Retry op queue should be empty"); + + if (source_addr) { + if (source_addr->shared_region) { + na_sm_queue_pair_release( + source_addr->shared_region, source_addr->queue_pair_idx); + + ret = na_sm_region_close(username, source_addr->pid, + source_addr->id, NA_TRUE, source_addr->shared_region); + NA_CHECK_NA_ERROR(done, ret, "na_sm_region_close() failed"); + } + if (source_addr->tx_notify > 0) { + int rc; + + ret = na_sm_poll_deregister( + na_sm_endpoint->poll_set, source_addr->tx_notify); + NA_CHECK_NA_ERROR(done, ret, "na_sm_poll_deregister() failed"); + + rc = hg_event_destroy(source_addr->tx_notify); + NA_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, + na_sm_errno_to_na(errno), "hg_event_destroy() failed"); + } + if (na_sm_endpoint->sock > 0) { + if (na_sm_endpoint->listen) { + ret = na_sm_poll_deregister( + na_sm_endpoint->poll_set, na_sm_endpoint->sock); + NA_CHECK_NA_ERROR(done, ret, "na_sm_poll_deregister() failed"); + } + ret = na_sm_sock_close(username, source_addr->pid, source_addr->id, + na_sm_endpoint->listen, na_sm_endpoint->sock); + NA_CHECK_NA_ERROR(done, ret, "na_sm_sock_close() failed"); + + na_sm_endpoint->sock = -1; + } + free(source_addr); + na_sm_endpoint->source_addr = NULL; + } + + if (na_sm_endpoint->poll_set) { + int rc = hg_poll_destroy(na_sm_endpoint->poll_set); + NA_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, + na_sm_errno_to_na(errno), "hg_poll_destroy() failed"); + + na_sm_endpoint->poll_set = NULL; + } + + /* Free hash table */ + if (na_sm_endpoint->addr_map.map) { + hg_hash_table_free(na_sm_endpoint->addr_map.map); + hg_thread_rwlock_destroy(&na_sm_endpoint->addr_map.lock); + } + + /* Destroy mutexes */ + hg_thread_spin_destroy(&na_sm_endpoint->unexpected_msg_queue.lock); + hg_thread_spin_destroy(&na_sm_endpoint->unexpected_op_queue.lock); + hg_thread_spin_destroy(&na_sm_endpoint->expected_op_queue.lock); + hg_thread_spin_destroy(&na_sm_endpoint->retry_op_queue.lock); + hg_thread_spin_destroy(&na_sm_endpoint->poll_addr_list.lock); done: return ret; } +/*---------------------------------------------------------------------------*/ +static NA_INLINE na_return_t +na_sm_queue_pair_reserve(struct na_sm_region *na_sm_region, na_uint8_t *index) +{ + unsigned int j = 0; + + do { + hg_util_int64_t bits = 1LL; + unsigned int i = 0; + + do { + hg_util_int64_t available = + hg_atomic_get64(&na_sm_region->available.val[j]); + if (!available) { + j++; + break; + } + + if ((available & bits) != bits) { + /* Already reserved */ + hg_atomic_fence(); + i++; + bits <<= 1; + continue; + } + + if (hg_atomic_cas64(&na_sm_region->available.val[j], available, + available & ~bits)) { +#ifdef NA_HAS_DEBUG + char buf[65] = {'\0'}; + available = hg_atomic_get64(&na_sm_region->available.val[j]); + NA_LOG_DEBUG("Reserved pair index %u\n### Available: %s", + (i + (j * 64)), + lltoa((hg_util_uint64_t) available, buf, 2)); #endif + *index = (na_uint8_t)(i + (j * 64)); + return NA_SUCCESS; + } + + /* Can't use atomic XOR directly, if there is a race and the cas + * fails, we should be able to pick the next one available */ + } while (i < 64); + } while (j < 4); + + return NA_AGAIN; +} + +/*---------------------------------------------------------------------------*/ +static NA_INLINE void +na_sm_queue_pair_release(struct na_sm_region *na_sm_region, na_uint8_t index) +{ + hg_atomic_or64(&na_sm_region->available.val[index / 64], 1LL << index % 64); + NA_LOG_DEBUG("Released pair index %u", index); +} + +/*---------------------------------------------------------------------------*/ +static NA_INLINE struct na_sm_addr * +na_sm_addr_map_lookup(struct na_sm_map *na_sm_map, na_uint64_t key) +{ + hg_hash_table_value_t value = NULL; + + /* Lookup key */ + hg_thread_rwlock_rdlock(&na_sm_map->lock); + value = hg_hash_table_lookup(na_sm_map->map, (hg_hash_table_key_t) &key); + hg_thread_rwlock_release_rdlock(&na_sm_map->lock); + + return (value == HG_HASH_TABLE_NULL) ? NULL : *(struct na_sm_addr **) value; +} /*---------------------------------------------------------------------------*/ static na_return_t -na_sm_poll_register(na_class_t *na_class, na_sm_poll_type_t poll_type, - struct na_sm_addr *na_sm_addr) +na_sm_addr_map_insert(struct na_sm_map *na_sm_map, na_uint64_t key, + na_return_t (*insert_cb)(void *, struct na_sm_addr **), void *arg, + struct na_sm_addr **addr) { - struct na_sm_poll_data *na_sm_poll_data = NULL; - struct na_sm_poll_data **na_sm_poll_data_ptr = NULL; - unsigned int flags = HG_POLLIN; - int fd, rc; + struct na_sm_addr *na_sm_addr = NULL; + hg_hash_table_value_t value = NULL; + hg_hash_table_key_t key_ptr = (hg_hash_table_key_t) &key; na_return_t ret = NA_SUCCESS; + na_bool_t inserted = NA_FALSE; + int rc; - switch (poll_type) { - case NA_SM_ACCEPT: - fd = na_sm_addr->sock; - na_sm_poll_data_ptr = &na_sm_addr->sock_poll_data; - break; - case NA_SM_SOCK: - fd = na_sm_addr->sock; - na_sm_poll_data_ptr = &na_sm_addr->sock_poll_data; - break; - case NA_SM_NOTIFY: - fd = na_sm_addr->local_notify; - na_sm_poll_data_ptr = &na_sm_addr->local_notify_poll_data; - break; - default: - NA_GOTO_ERROR(error, ret, NA_INVALID_ARG, "Invalid poll type"); + hg_thread_rwlock_wrlock(&na_sm_map->lock); + + /* Look up again to prevent race between lock release/acquire */ + value = hg_hash_table_lookup(na_sm_map->map, key_ptr); + if (value != HG_HASH_TABLE_NULL) { + ret = NA_EXIST; /* Entry already exists */ + goto done; } - na_sm_poll_data = - (struct na_sm_poll_data *) malloc(sizeof(struct na_sm_poll_data)); - NA_CHECK_ERROR(na_sm_poll_data == NULL, error, ret, NA_NOMEM, - "Could not allocate NA SM poll data"); - na_sm_poll_data->na_class = na_class; - na_sm_poll_data->type = poll_type; - na_sm_poll_data->addr = na_sm_addr; - *na_sm_poll_data_ptr = na_sm_poll_data; + /* Allocate new key */ + key_ptr = (hg_hash_table_key_t) malloc(sizeof(na_uint64_t)); + NA_CHECK_ERROR(key_ptr == NULL, error, ret, NA_NOMEM, + "Cannot allocate memory for new addr key"); + *((na_uint64_t *) key_ptr) = key; + + /* Allocate new value */ + value = (hg_hash_table_value_t) malloc(sizeof(struct na_sm_addr *)); + NA_CHECK_ERROR(value == NULL, error, ret, NA_NOMEM, + "cannot allocate memory for pointer to address"); + + /* Insert new value */ + rc = hg_hash_table_insert(na_sm_map->map, key_ptr, value); + NA_CHECK_ERROR( + rc == 0, error, ret, NA_NOMEM, "hg_hash_table_insert() failed"); + inserted = NA_TRUE; + + /* This is a new address, look it up */ + ret = insert_cb(arg, &na_sm_addr); + NA_CHECK_NA_ERROR(error, ret, "Could not execute insertion callback"); + + *((struct na_sm_addr **) value) = na_sm_addr; + +done: + hg_thread_rwlock_release_wrlock(&na_sm_map->lock); - rc = hg_poll_add(NA_SM_CLASS(na_class)->poll_set, fd, flags, - na_sm_progress_cb, na_sm_poll_data); - NA_CHECK_ERROR(rc != HG_UTIL_SUCCESS, error, ret, NA_PROTOCOL_ERROR, - "hg_poll_add() failed"); + *addr = *((struct na_sm_addr **) value); return ret; error: - free(na_sm_poll_data); - if (na_sm_poll_data_ptr) - *na_sm_poll_data_ptr = NULL; + if (inserted) { + rc = hg_hash_table_remove(na_sm_map->map, key_ptr); + NA_CHECK_ERROR_DONE(rc == 0, "Could not remove key"); + } else { + free(value); + free(key_ptr); + } + hg_thread_rwlock_release_wrlock(&na_sm_map->lock); return ret; } /*---------------------------------------------------------------------------*/ static na_return_t -na_sm_poll_deregister(na_class_t *na_class, na_sm_poll_type_t poll_type, - struct na_sm_addr *na_sm_addr) +na_sm_addr_lookup_insert_cb(void *arg, struct na_sm_addr **addr) { - int fd, rc; - struct na_sm_poll_data *na_sm_poll_data = NULL; + struct na_sm_lookup_args *args = (struct na_sm_lookup_args *) arg; + struct na_sm_addr *na_sm_addr = NULL; + na_uint8_t queue_pair_idx = 0; + na_sm_cmd_hdr_t cmd_hdr = {.val = 0}; + struct na_sm_region *shared_region = NULL; + na_bool_t queue_pair_reserved = NA_FALSE; + int tx_notify = -1, rx_notify = -1; na_return_t ret = NA_SUCCESS; + int rc; - switch (poll_type) { - case NA_SM_ACCEPT: - na_sm_poll_data = na_sm_addr->sock_poll_data; - fd = na_sm_addr->sock; - break; - case NA_SM_SOCK: - na_sm_poll_data = na_sm_addr->sock_poll_data; - fd = na_sm_addr->sock; - break; - case NA_SM_NOTIFY: - na_sm_poll_data = na_sm_addr->local_notify_poll_data; - fd = na_sm_addr->local_notify; - break; - default: - NA_GOTO_ERROR(done, ret, NA_INVALID_ARG, "Invalid poll type"); - } + /* Open shm region */ + ret = na_sm_region_open( + args->username, args->pid, args->id, NA_FALSE, &shared_region); + NA_CHECK_NA_ERROR(error, ret, "Could not open shared-memory region"); + + /* Reserve queue pair */ + ret = na_sm_queue_pair_reserve(shared_region, &queue_pair_idx); + NA_CHECK_NA_ERROR(error, ret, "Could not reserve queue pair"); + queue_pair_reserved = NA_TRUE; + + /* Fill cmd header */ + cmd_hdr.hdr.type = NA_SM_RESERVED; + cmd_hdr.hdr.pid = (unsigned int) args->endpoint->source_addr->pid; + cmd_hdr.hdr.id = args->endpoint->source_addr->id & 0xff; + cmd_hdr.hdr.pair_idx = queue_pair_idx & 0xff; + + /* Do not create signals if not waiting */ + if (args->endpoint->poll_set) { + /* Create tx event */ + ret = na_sm_event_create(args->username, args->pid, args->id, + queue_pair_idx, 't', &tx_notify); + NA_CHECK_NA_ERROR(error, ret, "Could not create event"); + + /* Create rx event */ + ret = na_sm_event_create(args->username, args->pid, args->id, + queue_pair_idx, 'r', &rx_notify); + NA_CHECK_NA_ERROR(error, ret, "Could not create event"); + + /* Send events to remote process */ + ret = na_sm_addr_event_send(args->endpoint->sock, args->username, + args->pid, args->id, cmd_hdr, tx_notify, rx_notify, NA_FALSE); + NA_CHECK_NA_ERROR(error, ret, "Could not send addr events"); + } else { + NA_LOG_DEBUG("Pushing cmd with %d for %d/%" SCNu8 "/%" SCNu8 " val=%lu", + cmd_hdr.hdr.type, cmd_hdr.hdr.pid, cmd_hdr.hdr.id, + cmd_hdr.hdr.pair_idx, cmd_hdr.val); - rc = hg_poll_remove(NA_SM_CLASS(na_class)->poll_set, fd); - NA_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, NA_PROTOCOL_ERROR, - "hg_poll_remove() failed"); - free(na_sm_poll_data); + /* TODO would be nice to also do that with poll set */ + rc = na_sm_cmd_queue_push(&shared_region->cmd_queue, cmd_hdr); + NA_CHECK_ERROR(rc == NA_FALSE, error, ret, NA_AGAIN, "Full queue"); + } -done: - return ret; -} + /* Allocate address */ + ret = na_sm_addr_create(args->endpoint, shared_region, args->pid, args->id, + queue_pair_idx, tx_notify, rx_notify, NA_FALSE, &na_sm_addr); + NA_CHECK_NA_ERROR(error, ret, "Could not allocate address"); -/*---------------------------------------------------------------------------*/ -static na_return_t -na_sm_setup_shm(na_class_t *na_class, struct na_sm_addr *na_sm_addr) -{ - char filename[NA_SM_MAX_FILENAME], pathname[NA_SM_MAX_FILENAME]; - struct na_sm_copy_buf *na_sm_copy_buf = NULL; - int listen_sock = -1; - na_return_t ret = NA_SUCCESS; + /* Add address to list of addresses to poll */ + hg_thread_spin_lock(&args->endpoint->poll_addr_list.lock); + HG_LIST_INSERT_HEAD( + &args->endpoint->poll_addr_list.list, na_sm_addr, entry); + hg_thread_spin_unlock(&args->endpoint->poll_addr_list.lock); - /* Create SHM buffer */ - NA_SM_GEN_SHM_NAME(filename, NA_SM_CLASS(na_class)->username, na_sm_addr); - na_sm_copy_buf = (struct na_sm_copy_buf *) na_sm_open_shared_buf( - filename, sizeof(struct na_sm_copy_buf), NA_TRUE); - NA_CHECK_ERROR(na_sm_copy_buf == NULL, error, ret, NA_PROTOCOL_ERROR, - "Could not create copy buffer"); + *addr = na_sm_addr; - /* Initialize copy buf, store 1111111111...1111 */ - hg_atomic_init64(&na_sm_copy_buf->available.val, ~((hg_util_int64_t)0)); - na_sm_addr->na_sm_copy_buf = na_sm_copy_buf; + return ret; - /* Create SHM sock */ - NA_SM_GEN_SOCK_PATH(pathname, NA_SM_CLASS(na_class)->username, na_sm_addr); - ret = na_sm_create_sock(pathname, NA_TRUE, &listen_sock); - NA_CHECK_NA_ERROR(error, ret, "Could not create sock"); - na_sm_addr->sock = listen_sock; +error: + if (shared_region) { + na_return_t err_ret; - /* Add listen_sock to poll set */ - ret = na_sm_poll_register(na_class, NA_SM_ACCEPT, na_sm_addr); - NA_CHECK_NA_ERROR(error, ret, "Could not add listen_sock to poll set"); + if (queue_pair_reserved) { + na_sm_queue_pair_release(shared_region, queue_pair_idx); - return ret; + if (tx_notify > 0) { + err_ret = na_sm_event_destroy(args->username, args->pid, + args->id, queue_pair_idx, 't', NA_TRUE, tx_notify); + NA_CHECK_ERROR_DONE( + err_ret != NA_SUCCESS, "na_sm_event_destroy() failed"); + } + if (rx_notify > 0) { + err_ret = na_sm_event_destroy(args->username, args->pid, + args->id, queue_pair_idx, 'r', NA_TRUE, rx_notify); + NA_CHECK_ERROR_DONE( + err_ret != NA_SUCCESS, "na_sm_event_destroy() failed"); + } + } -error: - if (listen_sock != -1) { - na_sm_close_sock(listen_sock, pathname); - na_sm_addr->sock = -1; - } - if (na_sm_copy_buf) { - na_sm_close_shared_buf(filename, na_sm_copy_buf, - sizeof(struct na_sm_copy_buf)); - na_sm_addr->na_sm_copy_buf = NULL; + err_ret = na_sm_region_close( + args->username, args->pid, args->id, NA_FALSE, shared_region); + NA_CHECK_ERROR_DONE( + err_ret != NA_SUCCESS, "Could not close shared-memory region"); } + return ret; } /*---------------------------------------------------------------------------*/ static na_return_t -na_sm_send_addr_info(na_class_t *na_class, struct na_sm_addr *na_sm_addr) +na_sm_addr_create(struct na_sm_endpoint *na_sm_endpoint, + struct na_sm_region *shared_region, pid_t pid, na_uint8_t id, + na_uint8_t queue_pair_idx, int tx_notify, int rx_notify, + na_bool_t unexpected, struct na_sm_addr **addr) { - struct msghdr msg = NA_SM_MSGHDR_INITIALIZER; - ssize_t nsend; - struct iovec iovec[2]; + struct na_sm_addr *na_sm_addr = NULL; na_return_t ret = NA_SUCCESS; - /* Send local PID / ID */ - iovec[0].iov_base = &NA_SM_CLASS(na_class)->self_addr->pid; - iovec[0].iov_len = sizeof(pid_t); - iovec[1].iov_base = &NA_SM_CLASS(na_class)->self_addr->id; - iovec[1].iov_len = sizeof(unsigned int); - msg.msg_iov = iovec; - msg.msg_iovlen = 2; + /* Allocate new addr */ + na_sm_addr = (struct na_sm_addr *) malloc(sizeof(struct na_sm_addr)); + NA_CHECK_ERROR(na_sm_addr == NULL, done, ret, NA_NOMEM, + "Could not allocate NA SM addr"); + memset(na_sm_addr, 0, sizeof(struct na_sm_addr)); + na_sm_addr->unexpected = unexpected; + hg_atomic_init32(&na_sm_addr->ref_count, 1); + + /* Assign PID/ID */ + na_sm_addr->pid = pid; + na_sm_addr->id = id; + + /* Assign queue pair index / shared region */ + na_sm_addr->queue_pair_idx = queue_pair_idx; + na_sm_addr->shared_region = shared_region; + + if (!unexpected) { + /* Simply assign queues (source address may not have shared region) */ + na_sm_addr->tx_queue = + shared_region ? &shared_region->queue_pairs[queue_pair_idx].tx_queue + : NULL; + na_sm_addr->rx_queue = + shared_region ? &shared_region->queue_pairs[queue_pair_idx].rx_queue + : NULL; + + /* Simply assign notify descriptors */ + na_sm_addr->tx_notify = tx_notify; + na_sm_addr->rx_notify = rx_notify; + } else { + /* Invert queues so that local rx is remote tx */ + na_sm_addr->tx_queue = + &shared_region->queue_pairs[queue_pair_idx].rx_queue; + na_sm_addr->rx_queue = + &shared_region->queue_pairs[queue_pair_idx].tx_queue; + + /* Invert descriptors so that local rx is remote tx */ + na_sm_addr->tx_notify = rx_notify; + na_sm_addr->rx_notify = tx_notify; + } + + if (na_sm_endpoint->poll_set && (na_sm_addr->rx_notify > 0)) { + NA_LOG_DEBUG( + "Registering rx notify %d for polling", na_sm_addr->rx_notify); + /* Add remote rx notify to poll set */ + ret = na_sm_poll_register(na_sm_endpoint->poll_set, + na_sm_addr->rx_notify, na_sm_progress_rx_notify, na_sm_addr); + NA_CHECK_NA_ERROR(done, ret, "Could not add rx notify to poll set"); + } - nsend = sendmsg(na_sm_addr->sock, &msg, 0); - NA_CHECK_ERROR(nsend == -1, done, ret, NA_PROTOCOL_ERROR, - "sendmsg() failed (%s)", strerror(errno)); + *addr = na_sm_addr; done: return ret; @@ -1443,32 +2236,70 @@ na_sm_send_addr_info(na_class_t *na_class, struct na_sm_addr *na_sm_addr) /*---------------------------------------------------------------------------*/ static na_return_t -na_sm_recv_addr_info(struct na_sm_addr *na_sm_addr, na_bool_t *received) +na_sm_addr_destroy(struct na_sm_endpoint *na_sm_endpoint, const char *username, + struct na_sm_addr *na_sm_addr) { - struct msghdr msg = NA_SM_MSGHDR_INITIALIZER; - ssize_t nrecv; - struct iovec iovec[2]; na_return_t ret = NA_SUCCESS; - /* Receive remote PID / ID */ - iovec[0].iov_base = &na_sm_addr->pid; - iovec[0].iov_len = sizeof(pid_t); - iovec[1].iov_base = &na_sm_addr->id; - iovec[1].iov_len = sizeof(unsigned int); - msg.msg_iov = iovec; - msg.msg_iovlen = 2; + if (na_sm_addr->unexpected) { + /* Release queue pair */ + na_sm_queue_pair_release( + na_sm_addr->shared_region, na_sm_addr->queue_pair_idx); + } else { + na_sm_cmd_hdr_t cmd_hdr = {.val = 0}; + + /* Fill cmd header */ + cmd_hdr.hdr.type = NA_SM_RELEASED; + cmd_hdr.hdr.pid = (unsigned int) na_sm_endpoint->source_addr->pid; + cmd_hdr.hdr.id = na_sm_endpoint->source_addr->id & 0xff; + cmd_hdr.hdr.pair_idx = na_sm_addr->queue_pair_idx & 0xff; + + /* Do not create signals if not waiting */ + if (na_sm_endpoint->poll_set) { + /* Send events to remote process (ignore error as this is best + * effort to clean up resources) */ + ret = na_sm_addr_event_send(na_sm_endpoint->sock, username, + na_sm_addr->pid, na_sm_addr->id, cmd_hdr, -1, -1, NA_TRUE); + NA_CHECK_NA_ERROR(done, ret, "Could not send addr events"); + } else { + na_bool_t rc; - nrecv = recvmsg(na_sm_addr->sock, &msg, 0); - if (nrecv == -1) { - if (likely(errno == EAGAIN)) { - *received = NA_FALSE; - goto done; - } else - NA_GOTO_ERROR(done, ret, NA_PROTOCOL_ERROR, "recvmsg() failed (%s)", - strerror(errno)); + NA_LOG_DEBUG("Pushing cmd with %d for %d/%" SCNu8 "/%" SCNu8 + " val=%lu", + cmd_hdr.hdr.type, cmd_hdr.hdr.pid, cmd_hdr.hdr.id, + cmd_hdr.hdr.pair_idx, cmd_hdr.val); + + /* TODO would be nice to also do that with poll set */ + rc = na_sm_cmd_queue_push( + &na_sm_addr->shared_region->cmd_queue, cmd_hdr); + NA_CHECK_ERROR(rc == NA_FALSE, done, ret, NA_AGAIN, "Full queue"); + } + + /* Close shared-memory region */ + ret = na_sm_region_close(username, na_sm_addr->pid, na_sm_addr->id, + NA_FALSE, na_sm_addr->shared_region); + NA_CHECK_NA_ERROR(done, ret, "Could not close shared-memory region"); } - *received = NA_TRUE; + if (na_sm_addr->tx_notify > 0) { + ret = na_sm_event_destroy(username, na_sm_addr->pid, na_sm_addr->id, + na_sm_addr->queue_pair_idx, 't', !na_sm_addr->unexpected, + na_sm_addr->tx_notify); + NA_CHECK_NA_ERROR(done, ret, "na_sm_event_destroy() failed"); + } + + if (na_sm_addr->rx_notify > 0) { + ret = na_sm_poll_deregister( + na_sm_endpoint->poll_set, na_sm_addr->rx_notify); + NA_CHECK_NA_ERROR(done, ret, "na_sm_poll_deregister() failed"); + + ret = na_sm_event_destroy(username, na_sm_addr->pid, na_sm_addr->id, + na_sm_addr->queue_pair_idx, 'r', !na_sm_addr->unexpected, + na_sm_addr->rx_notify); + NA_CHECK_NA_ERROR(done, ret, "na_sm_event_destroy() failed"); + } + + free(na_sm_addr); done: return ret; @@ -1476,12 +2307,15 @@ na_sm_recv_addr_info(struct na_sm_addr *na_sm_addr, na_bool_t *received) /*---------------------------------------------------------------------------*/ static na_return_t -na_sm_send_conn_id(struct na_sm_addr *na_sm_addr) +na_sm_addr_event_send(int sock, const char *username, pid_t pid, na_uint8_t id, + na_sm_cmd_hdr_t cmd_hdr, int tx_notify, int rx_notify, + na_bool_t ignore_error) { - struct msghdr msg = NA_SM_MSGHDR_INITIALIZER; + struct sockaddr_un addr; + struct msghdr msg; struct cmsghdr *cmsg; /* Contains the file descriptors to pass */ - int fds[2] = {na_sm_addr->local_notify, na_sm_addr->remote_notify}; + int fds[2] = {tx_notify, rx_notify}; union { /* ancillary data buffer, wrapped in a union in order to ensure it is suitably aligned */ @@ -1492,28 +2326,51 @@ na_sm_send_conn_id(struct na_sm_addr *na_sm_addr) struct iovec iovec[1]; ssize_t nsend; na_return_t ret = NA_SUCCESS; + int rc; + + /* Generate named socket path */ + memset(&addr, 0, sizeof(struct sockaddr_un)); + addr.sun_family = AF_UNIX; - /* Send local PID / ID */ - iovec[0].iov_base = &na_sm_addr->conn_id; - iovec[0].iov_len = sizeof(unsigned int); + rc = NA_SM_GEN_SOCK_PATH( + addr.sun_path, NA_SM_MAX_FILENAME, username, pid, id); + NA_CHECK_ERROR(rc < 0 || rc > NA_SM_MAX_FILENAME, done, ret, NA_OVERFLOW, + "NA_SM_GEN_SOCK_PATH() failed, rc: %d", rc); + strcat(addr.sun_path, NA_SM_SOCK_NAME); + + /* Set address of destination */ + msg.msg_name = &addr; + msg.msg_namelen = (socklen_t) SUN_LEN(&addr); + msg.msg_flags = 0; /* unused */ + + /* Send cmd */ + iovec[0].iov_base = &cmd_hdr; + iovec[0].iov_len = sizeof(cmd_hdr); msg.msg_iov = iovec; msg.msg_iovlen = 1; - /* Send notify event descriptors as ancillary data */ - msg.msg_control = u.buf; - msg.msg_controllen = sizeof(u.buf); - cmsg = CMSG_FIRSTHDR(&msg); - cmsg->cmsg_level = SOL_SOCKET; - cmsg->cmsg_type = SCM_RIGHTS; - cmsg->cmsg_len = CMSG_LEN(sizeof(fds)); - - /* Initialize the payload */ - fdptr = (int *) CMSG_DATA(cmsg); - memcpy(fdptr, fds, sizeof(fds)); + if (tx_notify > 0 && rx_notify > 0) { + /* Send notify event descriptors as ancillary data */ + msg.msg_control = u.buf; + msg.msg_controllen = sizeof(u.buf); + cmsg = CMSG_FIRSTHDR(&msg); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + cmsg->cmsg_len = CMSG_LEN(sizeof(fds)); + + /* Initialize the payload */ + fdptr = (int *) CMSG_DATA(cmsg); + memcpy(fdptr, fds, sizeof(fds)); + } else { + msg.msg_control = NULL; + msg.msg_controllen = 0; + } - nsend = sendmsg(na_sm_addr->sock, &msg, 0); - NA_CHECK_ERROR(nsend == -1, done, ret, NA_PROTOCOL_ERROR, - "sendmsg() failed (%s)", strerror(errno)); + nsend = sendmsg(sock, &msg, 0); + if (!ignore_error) { + NA_CHECK_ERROR(nsend == -1, done, ret, na_sm_errno_to_na(errno), + "sendmsg() failed (%s)", strerror(errno)); + } done: return ret; @@ -1521,9 +2378,10 @@ na_sm_send_conn_id(struct na_sm_addr *na_sm_addr) /*---------------------------------------------------------------------------*/ static na_return_t -na_sm_recv_conn_id(struct na_sm_addr *na_sm_addr, na_bool_t *received) +na_sm_addr_event_recv(int sock, na_sm_cmd_hdr_t *cmd_hdr, int *tx_notify, + int *rx_notify, na_bool_t *received) { - struct msghdr msg = NA_SM_MSGHDR_INITIALIZER; + struct msghdr msg; struct cmsghdr *cmsg; int *fdptr; int fds[2]; @@ -1537,9 +2395,14 @@ na_sm_recv_conn_id(struct na_sm_addr *na_sm_addr, na_bool_t *received) struct iovec iovec[1]; na_return_t ret = NA_SUCCESS; - /* Receive remote PID / ID */ - iovec[0].iov_base = &na_sm_addr->conn_id; - iovec[0].iov_len = sizeof(unsigned int); + /* Ignore address of source */ + msg.msg_name = NULL; + msg.msg_namelen = 0; + msg.msg_flags = 0; /* unused */ + + /* Recv reserved queue pair index */ + iovec[0].iov_base = cmd_hdr; + iovec[0].iov_len = sizeof(*cmd_hdr); msg.msg_iov = iovec; msg.msg_iovlen = 1; @@ -1547,91 +2410,45 @@ na_sm_recv_conn_id(struct na_sm_addr *na_sm_addr, na_bool_t *received) msg.msg_control = u.buf; msg.msg_controllen = sizeof u.buf; - nrecv = recvmsg(na_sm_addr->sock, &msg, 0); + nrecv = recvmsg(sock, &msg, 0); if (nrecv == -1) { if (likely(errno == EAGAIN)) { *received = NA_FALSE; goto done; } else - NA_GOTO_ERROR(done, ret, NA_PROTOCOL_ERROR, "recvmsg() failed (%s)", - strerror(errno)); + NA_GOTO_ERROR(done, ret, na_sm_errno_to_na(errno), + "recvmsg() failed (% s)", strerror(errno)); } *received = NA_TRUE; /* Retrieve ancillary data */ cmsg = CMSG_FIRSTHDR(&msg); - NA_CHECK_ERROR(cmsg == NULL, done, ret, NA_PROTOCOL_ERROR, "NULL cmsg"); + if (cmsg) { + fdptr = (int *) CMSG_DATA(cmsg); + memcpy(fds, fdptr, sizeof(fds)); - fdptr = (int *) CMSG_DATA(cmsg); - memcpy(fds, fdptr ,sizeof(fds)); - /* Invert descriptors so that local is remote and remote is local */ - na_sm_addr->local_notify = fds[1]; - na_sm_addr->remote_notify = fds[0]; + *tx_notify = fds[0]; + *rx_notify = fds[1]; + } else { + *tx_notify = -1; + *rx_notify = -1; + } done: return ret; } -/*---------------------------------------------------------------------------*/ -static void -na_sm_ring_buf_init(struct na_sm_ring_buf *na_sm_ring_buf) -{ - struct hg_atomic_queue *hg_atomic_queue = &na_sm_ring_buf->queue; - unsigned int count = NA_SM_NUM_BUFS; - - hg_atomic_queue->prod_size = hg_atomic_queue->cons_size = count; - hg_atomic_queue->prod_mask = hg_atomic_queue->cons_mask = count - 1; - hg_atomic_init32(&hg_atomic_queue->prod_head, 0); - hg_atomic_init32(&hg_atomic_queue->cons_head, 0); - hg_atomic_init32(&hg_atomic_queue->prod_tail, 0); - hg_atomic_init32(&hg_atomic_queue->cons_tail, 0); -} - -/*---------------------------------------------------------------------------*/ -static NA_INLINE na_bool_t -na_sm_ring_buf_push(struct na_sm_ring_buf *na_sm_ring_buf, - na_sm_cacheline_hdr_t na_sm_hdr) -{ - int rc = hg_atomic_queue_push(&na_sm_ring_buf->queue, - (void *) na_sm_hdr.val); - - return (likely(rc == HG_UTIL_SUCCESS)) ? NA_TRUE : NA_FALSE; -} - -/*---------------------------------------------------------------------------*/ -static NA_INLINE na_bool_t -na_sm_ring_buf_pop(struct na_sm_ring_buf *na_sm_ring_buf, - na_sm_cacheline_hdr_t *na_sm_hdr_ptr) -{ - na_sm_hdr_ptr->val = (na_uint64_t) hg_atomic_queue_pop_mc( - &na_sm_ring_buf->queue); - - return (likely(na_sm_hdr_ptr->val)) ? NA_TRUE : NA_FALSE; -} - -/*---------------------------------------------------------------------------*/ -static NA_INLINE na_bool_t -na_sm_ring_buf_is_empty(struct na_sm_ring_buf *na_sm_ring_buf) -{ - return hg_atomic_queue_is_empty(&na_sm_ring_buf->queue); -} - /*---------------------------------------------------------------------------*/ static NA_INLINE na_return_t -na_sm_reserve_and_copy_buf(na_class_t *na_class, - struct na_sm_copy_buf *na_sm_copy_buf, const void *buf, size_t buf_size, - unsigned int *idx_reserved) +na_sm_buf_reserve(struct na_sm_copy_buf *na_sm_copy_buf, unsigned int *index) { hg_util_int64_t bits = 1LL; - na_return_t ret = NA_AGAIN; unsigned int i = 0; - hg_thread_spin_lock(&NA_SM_CLASS(na_class)->copy_buf_lock); - do { - hg_util_int64_t available = hg_atomic_get64( - &na_sm_copy_buf->available.val); + hg_util_int64_t available = + hg_atomic_get64(&na_sm_copy_buf->available.val); if (!available) { /* Nothing available */ break; @@ -1644,89 +2461,50 @@ na_sm_reserve_and_copy_buf(na_class_t *na_class, continue; } - if (hg_atomic_cas64(&na_sm_copy_buf->available.val, available, - available & ~bits)) { - /* Reservation succeeded, copy buffer */ - memcpy(na_sm_copy_buf->buf[i], buf, buf_size); - *idx_reserved = i; - NA_LOG_DEBUG("Reserved bit index %u:\n%s", i, itoa((hg_util_uint64_t) - hg_atomic_get64(&na_sm_copy_buf->available.val), 2)); - ret = NA_SUCCESS; - break; + if (hg_atomic_cas64( + &na_sm_copy_buf->available.val, available, available & ~bits)) { +#ifdef NA_HAS_DEBUG + char buf[65] = {'\0'}; + available = hg_atomic_get64(&na_sm_copy_buf->available.val); + NA_LOG_DEBUG("Reserved bit index %u\n### Available: %s", i, + lltoa((hg_util_uint64_t) available, buf, 2)); +#endif + *index = i; + return NA_SUCCESS; } /* Can't use atomic XOR directly, if there is a race and the cas * fails, we should be able to pick the next one available */ - } while (i < (NA_SM_NUM_BUFS - 1)); + } while (i < NA_SM_NUM_BUFS); - hg_thread_spin_unlock(&NA_SM_CLASS(na_class)->copy_buf_lock); - - return ret; + return NA_AGAIN; } /*---------------------------------------------------------------------------*/ static NA_INLINE void -na_sm_copy_and_free_buf(na_class_t *na_class, - struct na_sm_copy_buf *na_sm_copy_buf, void *buf, size_t buf_size, - unsigned int idx_reserved) +na_sm_buf_release(struct na_sm_copy_buf *na_sm_copy_buf, unsigned int index) { - hg_thread_spin_lock(&NA_SM_CLASS(na_class)->copy_buf_lock); - memcpy(buf, na_sm_copy_buf->buf[idx_reserved], buf_size); - hg_atomic_or64(&na_sm_copy_buf->available.val, 1LL << idx_reserved); - hg_thread_spin_unlock(&NA_SM_CLASS(na_class)->copy_buf_lock); + hg_atomic_or64(&na_sm_copy_buf->available.val, 1LL << index); + NA_LOG_DEBUG("Released bit index %u", index); } /*---------------------------------------------------------------------------*/ static NA_INLINE void -na_sm_release_buf(struct na_sm_copy_buf *na_sm_copy_buf, - unsigned int idx_reserved) +na_sm_buf_copy_to(struct na_sm_copy_buf *na_sm_copy_buf, unsigned int index, + const void *src, size_t n) { - hg_atomic_or64(&na_sm_copy_buf->available.val, 1LL << idx_reserved); + hg_thread_spin_lock(&na_sm_copy_buf->buf_locks[index]); + memcpy(na_sm_copy_buf->buf[index], src, n); + hg_thread_spin_unlock(&na_sm_copy_buf->buf_locks[index]); } /*---------------------------------------------------------------------------*/ -static na_return_t -na_sm_msg_insert(na_class_t *na_class, struct na_sm_op_id *na_sm_op_id, - unsigned int idx_reserved) +static NA_INLINE void +na_sm_buf_copy_from(struct na_sm_copy_buf *na_sm_copy_buf, unsigned int index, + void *dest, size_t n) { - na_sm_cacheline_hdr_t na_sm_hdr; - na_return_t ret = NA_SUCCESS; - int rc; - - /* Post the SM send request */ - na_sm_hdr.hdr.type = na_sm_op_id->completion_data.callback_info.type; - na_sm_hdr.hdr.buf_idx = idx_reserved & 0xff; - na_sm_hdr.hdr.buf_size = na_sm_op_id->info.msg.buf_size & 0xffff; - na_sm_hdr.hdr.tag = na_sm_op_id->info.msg.tag; - rc = (int) na_sm_ring_buf_push(na_sm_op_id->na_sm_addr->na_sm_send_ring_buf, - na_sm_hdr); - NA_CHECK_ERROR(rc == NA_FALSE, done, ret, NA_PROTOCOL_ERROR, - "Full ring buffer"); - - if (!NA_SM_CLASS(na_class)->no_wait) { - /* Notify remote */ -#ifdef HG_UTIL_HAS_SYSEVENTFD_H - rc = hg_event_set(na_sm_op_id->na_sm_addr->remote_notify); - NA_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, NA_PROTOCOL_ERROR, - "Could not send completion notification"); -#else - ret = na_sm_event_set(na_sm_op_id->na_sm_addr->remote_notify); - NA_CHECK_NA_ERROR(done, ret, "Could not send completion notification"); -#endif - } - - /* Immediate completion, add directly to completion queue. */ - ret = na_sm_complete(na_sm_op_id); - NA_CHECK_NA_ERROR(done, ret, "Could not complete operation"); - - if (!NA_SM_CLASS(na_class)->no_wait) { - /* Notify local completion */ - rc = hg_event_set(NA_SM_CLASS(na_class)->self_addr->local_notify); - NA_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, NA_PROTOCOL_ERROR, - "Could not signal local completion"); - } - -done: - return ret; + hg_thread_spin_lock(&na_sm_copy_buf->buf_locks[index]); + memcpy(dest, na_sm_copy_buf->buf[index], n); + hg_thread_spin_unlock(&na_sm_copy_buf->buf_locks[index]); } /*---------------------------------------------------------------------------*/ @@ -1748,17 +2526,18 @@ na_sm_offset_translate(struct na_sm_mem_handle *mem_handle, na_offset_t offset, new_offset -= mem_handle->iov[i].iov_len; } - iov[0].iov_base = (char *) mem_handle->iov[new_start_index].iov_base + - new_offset; - iov[0].iov_len = NA_SM_MIN(remaining_len, - mem_handle->iov[new_start_index].iov_len - new_offset); + iov[0].iov_base = + (char *) mem_handle->iov[new_start_index].iov_base + new_offset; + iov[0].iov_len = MIN( + remaining_len, mem_handle->iov[new_start_index].iov_len - new_offset); remaining_len -= iov[0].iov_len; - for (i = 1; remaining_len && (i < mem_handle->iovcnt - new_start_index); i++) { + for (i = 1; remaining_len && (i < mem_handle->iovcnt - new_start_index); + i++) { iov[i].iov_base = mem_handle->iov[i + new_start_index].iov_base; /* Can only transfer smallest size */ - iov[i].iov_len = NA_SM_MIN(remaining_len, - mem_handle->iov[i + new_start_index].iov_len); + iov[i].iov_len = + MIN(remaining_len, mem_handle->iov[i + new_start_index].iov_len); /* Decrease remaining len from the len of data */ remaining_len -= iov[i].iov_len; @@ -1769,403 +2548,203 @@ na_sm_offset_translate(struct na_sm_mem_handle *mem_handle, na_offset_t offset, /*---------------------------------------------------------------------------*/ static int -na_sm_progress_cb(void *arg, int error, hg_util_bool_t *progressed) +na_sm_progress_sock(void *arg, int error, struct hg_poll_event *event) { - na_class_t *na_class; - struct na_sm_poll_data *na_sm_poll_data = (struct na_sm_poll_data *) arg; - na_return_t ret; - - NA_CHECK_ERROR_NORET(na_sm_poll_data == NULL, error, "NULL SM poll data"); + struct na_sm_endpoint *na_sm_endpoint = (struct na_sm_endpoint *) arg; + struct na_sm_class *na_sm_class = + container_of(na_sm_endpoint, struct na_sm_class, endpoint); + na_sm_cmd_hdr_t cmd_hdr = {.val = 0}; + int tx_notify, rx_notify; + na_bool_t progressed = NA_FALSE; + na_return_t ret = NA_SUCCESS; - na_class = na_sm_poll_data->na_class; + NA_CHECK_ERROR(error, done, ret, NA_FAULT, "Unexpected poll error"); - if (error) { - ret = na_sm_progress_error(na_class, na_sm_poll_data->addr); - NA_CHECK_ERROR_NORET(ret != NA_SUCCESS, error, - "Could not process error"); - } else switch (na_sm_poll_data->type) { - case NA_SM_ACCEPT: - ret = na_sm_progress_accept(na_class, na_sm_poll_data->addr, - (hg_util_bool_t *) progressed); - NA_CHECK_ERROR_NORET(ret != NA_SUCCESS, error, - "Could not make progress on accept"); - break; - case NA_SM_SOCK: - if (na_sm_poll_data->addr != NA_SM_CLASS(na_class)->self_addr) { - ret = na_sm_progress_sock(na_class, na_sm_poll_data->addr, - (hg_util_bool_t *) progressed); - NA_CHECK_ERROR_NORET(ret != NA_SUCCESS, error, - "Could not make progress on sock"); - } - break; - case NA_SM_NOTIFY: - ret = na_sm_progress_notify(na_class, na_sm_poll_data->addr, - (hg_util_bool_t *) progressed); - NA_CHECK_ERROR_NORET(ret != NA_SUCCESS, error, - "Could not make progress on notify"); - break; - default: - NA_LOG_ERROR("Unknown poll data type"); - goto error; + /* Attempt to receive addr info (events, queue index) */ + ret = na_sm_addr_event_recv( + na_sm_endpoint->sock, &cmd_hdr, &tx_notify, &rx_notify, &progressed); + NA_CHECK_NA_ERROR(done, ret, "Could not recv addr events"); + if (!progressed) { + event->progressed = HG_UTIL_FALSE; + goto done; } - return HG_UTIL_SUCCESS; + /* Process received cmd, TODO would be nice to use cmd queue */ + ret = na_sm_process_cmd( + na_sm_endpoint, na_sm_class->username, cmd_hdr, tx_notify, rx_notify); + NA_CHECK_NA_ERROR(done, ret, "Could not process cmd"); -error: - return HG_UTIL_FAIL; + event->progressed = HG_UTIL_TRUE; + event->ptr = NULL; + +done: + return (ret == NA_SUCCESS) ? HG_UTIL_SUCCESS : HG_UTIL_FAIL; } /*---------------------------------------------------------------------------*/ static na_return_t -na_sm_progress_error(na_class_t *na_class, struct na_sm_addr *poll_addr) +na_sm_process_cmd(struct na_sm_endpoint *na_sm_endpoint, const char *username, + na_sm_cmd_hdr_t cmd_hdr, int tx_notify, int rx_notify) { na_return_t ret = NA_SUCCESS; - NA_CHECK_ERROR(poll_addr == NA_SM_CLASS(na_class)->self_addr, - done, ret, NA_PROTOCOL_ERROR, "Unsupported error occurred"); + NA_LOG_DEBUG("Processing cmd with %d from %d/%" SCNu8 "/%" SCNu8 " val=%lu", + cmd_hdr.hdr.type, cmd_hdr.hdr.pid, cmd_hdr.hdr.id & 0xff, + cmd_hdr.hdr.pair_idx & 0xff, cmd_hdr.val); + + switch (cmd_hdr.hdr.type) { + case NA_SM_RESERVED: { + struct na_sm_addr *na_sm_addr = NULL; + + /* Allocate source address */ + ret = na_sm_addr_create(na_sm_endpoint, + na_sm_endpoint->source_addr->shared_region, + (pid_t) cmd_hdr.hdr.pid, cmd_hdr.hdr.id, cmd_hdr.hdr.pair_idx, + tx_notify, rx_notify, NA_TRUE, &na_sm_addr); + NA_CHECK_NA_ERROR( + done, ret, "Could not allocate unexpected address"); + + /* Add address to list of addresses to poll */ + hg_thread_spin_lock(&na_sm_endpoint->poll_addr_list.lock); + HG_LIST_INSERT_HEAD( + &na_sm_endpoint->poll_addr_list.list, na_sm_addr, entry); + hg_thread_spin_unlock(&na_sm_endpoint->poll_addr_list.lock); + break; + } + case NA_SM_RELEASED: { + struct na_sm_addr *na_sm_addr = NULL; + na_bool_t found = NA_FALSE; + + /* Find address from list of addresses to poll */ + hg_thread_spin_lock(&na_sm_endpoint->poll_addr_list.lock); + HG_LIST_FOREACH ( + na_sm_addr, &na_sm_endpoint->poll_addr_list.list, entry) { + if ((na_sm_addr->queue_pair_idx == cmd_hdr.hdr.pair_idx) && + (na_sm_addr->pid == (pid_t) cmd_hdr.hdr.pid) && + (na_sm_addr->id == cmd_hdr.hdr.id)) { + found = NA_TRUE; + break; + } + } + hg_thread_spin_unlock(&na_sm_endpoint->poll_addr_list.lock); + + if (!found) { + /* Silently ignore if not found */ + NA_LOG_DEBUG( + "Could not find address for PID=%d, ID=%u, pair_index=%u", + cmd_hdr.hdr.pid, cmd_hdr.hdr.id, cmd_hdr.hdr.pair_idx); + break; + } + + if (hg_atomic_decr32(&na_sm_addr->ref_count)) + /* Cannot free yet */ + break; + + NA_LOG_DEBUG("Freeing addr for PID=%d, ID=%d", na_sm_addr->pid, + na_sm_addr->id); + + /* Remove address from list of addresses to poll */ + hg_thread_spin_lock(&na_sm_endpoint->poll_addr_list.lock); + HG_LIST_REMOVE(na_sm_addr, entry); + hg_thread_spin_unlock(&na_sm_endpoint->poll_addr_list.lock); - /* Handle case of peer disconnection */ - ret = na_sm_addr_free(na_class, poll_addr); + /* Destroy source address */ + ret = na_sm_addr_destroy(na_sm_endpoint, username, na_sm_addr); + NA_CHECK_NA_ERROR( + done, ret, "Could not allocate unexpected address"); + + break; + } + default: + NA_GOTO_ERROR( + done, ret, NA_INVALID_ARG, "Unknown type of operation"); + } done: return ret; } /*---------------------------------------------------------------------------*/ -static na_return_t -na_sm_progress_accept(na_class_t *na_class, struct na_sm_addr *poll_addr, - na_bool_t *progressed) +static int +na_sm_progress_tx_notify(void *arg, int error, struct hg_poll_event *event) { - struct na_sm_addr *na_sm_addr = NULL; - struct na_sm_ring_buf *na_sm_ring_buf = NULL; - char filename[NA_SM_MAX_FILENAME]; - int conn_sock, local_notify, remote_notify; - hg_time_t now; - double elapsed_ms; + struct na_sm_addr *poll_addr = (struct na_sm_addr *) arg; na_return_t ret = NA_SUCCESS; -#ifndef SOCK_NONBLOCK int rc; -#endif - NA_CHECK_ERROR(poll_addr != NA_SM_CLASS(na_class)->self_addr, - done, ret, NA_PROTOCOL_ERROR, "Unrecognized poll addr"); + NA_CHECK_ERROR(error, done, ret, NA_FAULT, "Unexpected poll error"); - /* Prevent from entering accept too often */ - hg_time_get_current(&now); - elapsed_ms = hg_time_to_double(hg_time_subtract(now, - NA_SM_CLASS(na_class)->last_accept_time)) * 1000.0; - if (elapsed_ms < NA_SM_ACCEPT_INTERVAL) { - *progressed = NA_FALSE; + /* Local notification only */ + rc = hg_event_get(poll_addr->tx_notify, &event->progressed); + NA_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, na_sm_errno_to_na(errno), + "Could not get completion notification"); + if (!event->progressed) goto done; - } - NA_SM_CLASS(na_class)->last_accept_time = now; - -#ifdef SOCK_NONBLOCK - conn_sock = accept4(poll_addr->sock, NULL, NULL, SOCK_NONBLOCK); -#else - conn_sock = accept(poll_addr->sock, NULL, NULL); -#endif - if (conn_sock == -1) { - if (likely(errno == EAGAIN)) { - *progressed = NA_FALSE; - goto done; - } else - NA_GOTO_ERROR(done, ret, NA_PROTOCOL_ERROR, "accept() failed (%s)", - strerror(errno)); - } -#ifndef SOCK_NONBLOCK - rc = fcntl(conn_sock, F_SETFL, O_NONBLOCK); - NA_CHECK_ERROR(rc == -1, done, ret, NA_PROTOCOL_ERROR, - "fcntl() failed (%s)", strerror(errno)); -#endif - - /* Allocate new addr and pass it to poll set */ - na_sm_addr = (struct na_sm_addr *) malloc(sizeof(struct na_sm_addr)); - NA_CHECK_ERROR(na_sm_addr == NULL, done, ret, NA_NOMEM, - "Could not allocate NA SM addr"); - - memset(na_sm_addr, 0, sizeof(struct na_sm_addr)); - hg_atomic_init32(&na_sm_addr->ref_count, 1); - na_sm_addr->accepted = NA_TRUE; - na_sm_addr->na_sm_copy_buf = poll_addr->na_sm_copy_buf; - na_sm_addr->sock = conn_sock; - /* We need to receive addr info in sock progress */ - na_sm_addr->sock_progress = NA_SM_ADDR_INFO; - - /* Add conn_sock to poll set */ - ret = na_sm_poll_register(na_class, NA_SM_SOCK, na_sm_addr); - NA_CHECK_NA_ERROR(done, ret, "Could not add conn_sock to poll set"); - - /* Set up ring buffer pair (send/recv) for connection IDs */ - na_sm_addr->conn_id = NA_SM_CLASS(na_class)->self_addr->conn_id; - NA_SM_GEN_RING_NAME(filename, NA_SM_SEND_NAME, - NA_SM_CLASS(na_class)->username, NA_SM_CLASS(na_class)->self_addr); - na_sm_ring_buf = (struct na_sm_ring_buf *) na_sm_open_shared_buf(filename, - NA_SM_RING_BUF_SIZE, NA_TRUE); - NA_CHECK_ERROR(na_sm_ring_buf == NULL, done, ret, NA_PROTOCOL_ERROR, - "Could not open ring buf"); - - /* Initialize ring buffer */ - na_sm_ring_buf_init(na_sm_ring_buf); - na_sm_addr->na_sm_send_ring_buf = na_sm_ring_buf; - - NA_SM_GEN_RING_NAME(filename, NA_SM_RECV_NAME, - NA_SM_CLASS(na_class)->username, NA_SM_CLASS(na_class)->self_addr); - na_sm_ring_buf = (struct na_sm_ring_buf *) na_sm_open_shared_buf(filename, - NA_SM_RING_BUF_SIZE, NA_TRUE); - NA_CHECK_ERROR(na_sm_ring_buf == NULL, done, ret, NA_PROTOCOL_ERROR, - "Could not open ring buf"); - - /* Initialize ring buffer */ - na_sm_ring_buf_init(na_sm_ring_buf); - na_sm_addr->na_sm_recv_ring_buf = na_sm_ring_buf; - - /* Create local signal event */ -#ifdef HG_UTIL_HAS_SYSEVENTFD_H - local_notify = hg_event_create(); - NA_CHECK_ERROR(local_notify == -1, done, ret, NA_PROTOCOL_ERROR, - "hg_event_create() failed"); -#else - /** - * If eventfd is not supported, we need to explicitly use named pipes in - * this case as kqueue file descriptors cannot be exchanged through - * ancillary data - */ - NA_SM_GEN_FIFO_NAME(filename, NA_SM_RECV_NAME, - NA_SM_CLASS(na_class)->username, NA_SM_CLASS(na_class)->self_addr); - local_notify = na_sm_event_create(filename); - NA_CHECK_ERROR(local_notify == -1, done, ret, NA_PROTOCOL_ERROR, - "na_sm_event_create() failed"); -#endif - na_sm_addr->local_notify = local_notify; - - /* Create remote signal event */ -#ifdef HG_UTIL_HAS_SYSEVENTFD_H - remote_notify = hg_event_create(); - NA_CHECK_ERROR(remote_notify == -1, done, ret, NA_PROTOCOL_ERROR, - "hg_event_create() failed"); -#else - /** - * If eventfd is not supported, we need to explicitly use named pipes in - * this case as kqueue file descriptors cannot be exchanged through - * ancillary data - */ - NA_SM_GEN_FIFO_NAME(filename, NA_SM_SEND_NAME, - NA_SM_CLASS(na_class)->username, NA_SM_CLASS(na_class)->self_addr); - remote_notify = na_sm_event_create(filename); - NA_CHECK_ERROR(remote_notify == -1, done, ret, NA_PROTOCOL_ERROR, - "na_sm_event_create() failed"); -#endif - na_sm_addr->remote_notify = remote_notify; - - /* Add local notify to poll set */ - ret = na_sm_poll_register(na_class, NA_SM_NOTIFY, na_sm_addr); - NA_CHECK_NA_ERROR(done, ret, "Could not add notify to poll set"); - /* Send connection ID / event IDs */ - ret = na_sm_send_conn_id(na_sm_addr); - NA_CHECK_NA_ERROR(done, ret, "Could not send connection ID"); + NA_LOG_DEBUG("Progressed tx notify %d", poll_addr->tx_notify); - /* Increment connection ID */ - NA_SM_CLASS(na_class)->self_addr->conn_id++; - - /* Push the addr to accepted addr queue so that we can free it later */ - hg_thread_spin_lock(&NA_SM_CLASS(na_class)->accepted_addr_queue_lock); - HG_QUEUE_PUSH_TAIL(&NA_SM_CLASS(na_class)->accepted_addr_queue, na_sm_addr, - entry); - hg_thread_spin_unlock(&NA_SM_CLASS(na_class)->accepted_addr_queue_lock); - - *progressed = NA_TRUE; + event->ptr = NULL; done: - return ret; - -//error: -// TODO -// return ret; + return (ret == NA_SUCCESS) ? HG_UTIL_SUCCESS : HG_UTIL_FAIL; } /*---------------------------------------------------------------------------*/ -static na_return_t -na_sm_progress_sock(na_class_t *na_class, struct na_sm_addr *poll_addr, - na_bool_t *progressed) +static int +na_sm_progress_rx_notify(void *arg, int error, struct hg_poll_event *event) { + struct na_sm_addr *poll_addr = (struct na_sm_addr *) arg; na_return_t ret = NA_SUCCESS; - NA_CHECK_ERROR(poll_addr == NA_SM_CLASS(na_class)->self_addr, - done, ret, NA_PROTOCOL_ERROR, "Unrecognized poll addr"); - - switch (poll_addr->sock_progress) { - case NA_SM_ADDR_INFO: { - na_bool_t received = NA_FALSE; + NA_CHECK_ERROR(error, done, ret, NA_FAULT, "Unexpected poll error"); - /* Receive addr info (PID / ID) */ - ret = na_sm_recv_addr_info(poll_addr, &received); - NA_CHECK_NA_ERROR(done, ret, "Could not recv addr info"); - if (!received) { - *progressed = NA_FALSE; - goto done; - } - - poll_addr->sock_progress = NA_SM_SOCK_DONE; - - /* Add addr to poll addr queue */ - hg_thread_spin_lock(&NA_SM_CLASS(na_class)->poll_addr_queue_lock); - HG_QUEUE_PUSH_TAIL(&NA_SM_CLASS(na_class)->poll_addr_queue, - poll_addr, poll_entry); - hg_thread_spin_unlock(&NA_SM_CLASS(na_class)->poll_addr_queue_lock); + /* Remote notification only */ + ret = na_sm_event_get(poll_addr->rx_notify, &event->progressed); + NA_CHECK_NA_ERROR(done, ret, "Could not get completion notification"); + if (!event->progressed) + goto done; - /* Progressed */ - *progressed = NA_TRUE; - } - break; - case NA_SM_CONN_ID: { - char filename[NA_SM_MAX_FILENAME]; - struct na_sm_ring_buf *na_sm_ring_buf; - struct na_sm_op_id *na_sm_op_id = NULL; - na_bool_t received = NA_FALSE; - - /* Receive connection ID / event IDs */ - ret = na_sm_recv_conn_id(poll_addr, &received); - NA_CHECK_NA_ERROR(done, ret, "Could not recv connection ID"); - if (!received) { - *progressed = NA_FALSE; - goto done; - } - poll_addr->sock_progress = NA_SM_SOCK_DONE; - - /* Find op ID that corresponds to addr */ - hg_thread_spin_lock(&NA_SM_CLASS(na_class)->lookup_op_queue_lock); - HG_QUEUE_FOREACH(na_sm_op_id, - &NA_SM_CLASS(na_class)->lookup_op_queue, entry) { - if (na_sm_op_id->na_sm_addr == poll_addr) { - HG_QUEUE_REMOVE(&NA_SM_CLASS(na_class)->lookup_op_queue, - na_sm_op_id, na_sm_op_id, entry); - break; - } - } - hg_thread_spin_unlock(&NA_SM_CLASS(na_class)->lookup_op_queue_lock); - - NA_CHECK_ERROR(na_sm_op_id == NULL, done, ret, NA_PROTOCOL_ERROR, - "Could not find lookup op ID, conn ID=%u, PID=%u", - poll_addr->conn_id, (unsigned int) poll_addr->pid); - - /* Open remote ring buf pair (send and recv names correspond to - * remote ring buffer pair) */ - NA_SM_GEN_RING_NAME(filename, NA_SM_RECV_NAME, - NA_SM_CLASS(na_class)->username, poll_addr); - na_sm_ring_buf = (struct na_sm_ring_buf *) na_sm_open_shared_buf( - filename, NA_SM_RING_BUF_SIZE, NA_FALSE); - NA_CHECK_ERROR(na_sm_ring_buf == NULL, done, ret, NA_PROTOCOL_ERROR, - "Could not open ring buf"); - poll_addr->na_sm_send_ring_buf = na_sm_ring_buf; - - NA_SM_GEN_RING_NAME(filename, NA_SM_SEND_NAME, - NA_SM_CLASS(na_class)->username, poll_addr); - na_sm_ring_buf = (struct na_sm_ring_buf *) na_sm_open_shared_buf( - filename, NA_SM_RING_BUF_SIZE, NA_FALSE); - NA_CHECK_ERROR(na_sm_ring_buf == NULL, done, ret, NA_PROTOCOL_ERROR, - "Could not open ring buf"); - poll_addr->na_sm_recv_ring_buf = na_sm_ring_buf; - - /* Add received local notify to poll set */ - ret = na_sm_poll_register(na_class, NA_SM_NOTIFY, poll_addr); - NA_CHECK_NA_ERROR(done, ret, "Could not add notify to poll set"); - - /* Add addr to poll addr queue */ - hg_thread_spin_lock(&NA_SM_CLASS(na_class)->poll_addr_queue_lock); - HG_QUEUE_PUSH_TAIL(&NA_SM_CLASS(na_class)->poll_addr_queue, - poll_addr, poll_entry); - hg_thread_spin_unlock(&NA_SM_CLASS(na_class)->poll_addr_queue_lock); - - /* Completion */ - ret = na_sm_complete(na_sm_op_id); - NA_CHECK_NA_ERROR(done, ret, "Could not complete operation"); + NA_LOG_DEBUG("Progressed rx notify %d", poll_addr->rx_notify); - /* Progressed */ - *progressed = NA_TRUE; - } - break; - default: - /* TODO Silently ignore, no progress */ - *progressed = NA_FALSE; - break; - } + event->ptr = poll_addr; done: - return ret; - -//error: -// TODO -// return ret; + return (ret == NA_SUCCESS) ? HG_UTIL_SUCCESS : HG_UTIL_FAIL; } /*---------------------------------------------------------------------------*/ static na_return_t -na_sm_progress_notify(na_class_t *na_class, struct na_sm_addr *poll_addr, - na_bool_t *progressed) +na_sm_progress_rx_queue(struct na_sm_endpoint *na_sm_endpoint, + struct na_sm_addr *poll_addr, na_bool_t *progressed) { - na_sm_cacheline_hdr_t na_sm_hdr; + na_sm_msg_hdr_t msg_hdr; na_return_t ret = NA_SUCCESS; - int rc; - - if (poll_addr == NA_SM_CLASS(na_class)->self_addr) { - /* Local notification */ - if (!NA_SM_CLASS(na_class)->no_wait) { - rc = hg_event_get(poll_addr->local_notify, - (hg_util_bool_t *) progressed); - NA_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, NA_PROTOCOL_ERROR, - "Could not get completion notification"); - } else - *progressed = NA_FALSE; - goto done; - } - - /* Remote notification */ - if (!NA_SM_CLASS(na_class)->no_wait) { - na_bool_t notified = NA_FALSE; - -#ifdef HG_UTIL_HAS_SYSEVENTFD_H - rc = hg_event_get(poll_addr->local_notify, - (hg_util_bool_t *) ¬ified); - NA_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, NA_PROTOCOL_ERROR, - "Could not get completion notification"); -#else - ret = na_sm_event_get(poll_addr->local_notify, ¬ified); - NA_CHECK_NA_ERROR(done, ret, "Could not get completion notification"); -#endif - if (!notified) { - *progressed = NA_FALSE; - goto done; - } - } - if (!na_sm_ring_buf_pop(poll_addr->na_sm_recv_ring_buf, &na_sm_hdr)) { + /* Look for message in rx queue */ + if (!na_sm_msg_queue_pop(poll_addr->rx_queue, &msg_hdr)) { *progressed = NA_FALSE; goto done; } - /* Progress expected and unexpected messages */ - switch (na_sm_hdr.hdr.type) { + NA_LOG_DEBUG("Found msg in queue"); + + /* Process expected and unexpected messages */ + switch (msg_hdr.hdr.type) { case NA_CB_SEND_UNEXPECTED: - ret = na_sm_progress_unexpected(na_class, poll_addr, na_sm_hdr); - NA_CHECK_NA_ERROR(done, ret, - "Could not make progress on unexpected msg"); + ret = na_sm_process_unexpected(&na_sm_endpoint->unexpected_op_queue, + poll_addr, msg_hdr, &na_sm_endpoint->unexpected_msg_queue); + NA_CHECK_NA_ERROR( + done, ret, "Could not make progress on unexpected msg"); break; case NA_CB_SEND_EXPECTED: - ret = na_sm_progress_expected(na_class, poll_addr, na_sm_hdr); - NA_CHECK_NA_ERROR(done, ret, - "Could not make progress on expected msg"); + ret = na_sm_process_expected( + &na_sm_endpoint->expected_op_queue, poll_addr, msg_hdr); + NA_CHECK_NA_ERROR( + done, ret, "Could not make progress on expected msg"); break; default: - NA_GOTO_ERROR(done, ret, NA_PROTOCOL_ERROR, - "Unknown type of operation"); - } - - /* Progress retries */ - if (!NA_SM_CLASS(na_class)->no_retry) { - ret = na_sm_progress_retries(na_class); - NA_CHECK_NA_ERROR(done, ret, "Could not make progress on retried msgs"); + NA_GOTO_ERROR( + done, ret, NA_INVALID_ARG, "Unknown type of operation"); } *progressed = NA_TRUE; @@ -2176,38 +2755,44 @@ na_sm_progress_notify(na_class_t *na_class, struct na_sm_addr *poll_addr, /*---------------------------------------------------------------------------*/ static na_return_t -na_sm_progress_unexpected(na_class_t *na_class, struct na_sm_addr *poll_addr, - na_sm_cacheline_hdr_t na_sm_hdr) +na_sm_process_unexpected(struct na_sm_op_queue *unexpected_op_queue, + struct na_sm_addr *poll_addr, na_sm_msg_hdr_t msg_hdr, + struct na_sm_unexpected_msg_queue *unexpected_msg_queue) { + struct na_sm_unexpected_info *na_sm_unexpected_info = NULL; struct na_sm_op_id *na_sm_op_id = NULL; na_return_t ret = NA_SUCCESS; + NA_LOG_DEBUG("Processing unexpected msg"); + /* Pop op ID from queue */ - hg_thread_spin_lock(&NA_SM_CLASS(na_class)->unexpected_op_queue_lock); - na_sm_op_id = HG_QUEUE_FIRST(&NA_SM_CLASS(na_class)->unexpected_op_queue); - HG_QUEUE_POP_HEAD(&NA_SM_CLASS(na_class)->unexpected_op_queue, entry); + hg_thread_spin_lock(&unexpected_op_queue->lock); + na_sm_op_id = HG_QUEUE_FIRST(&unexpected_op_queue->queue); + HG_QUEUE_POP_HEAD(&unexpected_op_queue->queue, entry); hg_atomic_and32(&na_sm_op_id->status, ~NA_SM_OP_QUEUED); - hg_thread_spin_unlock(&NA_SM_CLASS(na_class)->unexpected_op_queue_lock); + hg_thread_spin_unlock(&unexpected_op_queue->lock); if (likely(na_sm_op_id)) { /* Fill info */ na_sm_op_id->na_sm_addr = poll_addr; hg_atomic_incr32(&na_sm_op_id->na_sm_addr->ref_count); na_sm_op_id->info.msg.actual_buf_size = - (na_size_t) na_sm_hdr.hdr.buf_size; - na_sm_op_id->info.msg.tag = (na_tag_t) na_sm_hdr.hdr.tag; + (na_size_t) msg_hdr.hdr.buf_size; + na_sm_op_id->info.msg.tag = (na_tag_t) msg_hdr.hdr.tag; + + /* Copy buffer */ + na_sm_buf_copy_from(&poll_addr->shared_region->copy_bufs, + msg_hdr.hdr.buf_idx, na_sm_op_id->info.msg.buf.ptr, + msg_hdr.hdr.buf_size); - /* Copy and free buffer atomically */ - na_sm_copy_and_free_buf(na_class, poll_addr->na_sm_copy_buf, - na_sm_op_id->info.msg.buf.ptr, na_sm_hdr.hdr.buf_size, - na_sm_hdr.hdr.buf_idx); + /* Release buffer */ + na_sm_buf_release( + &poll_addr->shared_region->copy_bufs, msg_hdr.hdr.buf_idx); - /* Complete operation */ - ret = na_sm_complete(na_sm_op_id); + /* Complete operation (no need to notify) */ + ret = na_sm_complete(na_sm_op_id, 0); NA_CHECK_NA_ERROR(done, ret, "Could not complete operation"); } else { - struct na_sm_unexpected_info *na_sm_unexpected_info = NULL; - /* If no error and message arrived, keep a copy of the struct in * the unexpected message queue (should rarely happen) */ na_sm_unexpected_info = (struct na_sm_unexpected_info *) malloc( @@ -2216,68 +2801,79 @@ na_sm_progress_unexpected(na_class_t *na_class, struct na_sm_addr *poll_addr, "Could not allocate unexpected info"); na_sm_unexpected_info->na_sm_addr = poll_addr; - na_sm_unexpected_info->buf_size = (na_size_t) na_sm_hdr.hdr.buf_size; - na_sm_unexpected_info->tag = (na_tag_t) na_sm_hdr.hdr.tag; + na_sm_unexpected_info->buf_size = (na_size_t) msg_hdr.hdr.buf_size; + na_sm_unexpected_info->tag = (na_tag_t) msg_hdr.hdr.tag; /* Allocate buf */ na_sm_unexpected_info->buf = malloc(na_sm_unexpected_info->buf_size); - NA_CHECK_ERROR(na_sm_unexpected_info->buf == NULL, done, ret, NA_NOMEM, + NA_CHECK_ERROR(na_sm_unexpected_info->buf == NULL, error, ret, NA_NOMEM, "Could not allocate na_sm_unexpected_info buf"); - /* Copy and free buffer atomically */ - na_sm_copy_and_free_buf(na_class, poll_addr->na_sm_copy_buf, - na_sm_unexpected_info->buf, na_sm_hdr.hdr.buf_size, - na_sm_hdr.hdr.buf_idx); + /* Copy buffer */ + na_sm_buf_copy_from(&poll_addr->shared_region->copy_bufs, + msg_hdr.hdr.buf_idx, na_sm_unexpected_info->buf, + msg_hdr.hdr.buf_size); + + /* Release buffer */ + na_sm_buf_release( + &poll_addr->shared_region->copy_bufs, msg_hdr.hdr.buf_idx); /* Otherwise push the unexpected message into our unexpected queue so * that we can treat it later when a recv_unexpected is posted */ - hg_thread_spin_lock(&NA_SM_CLASS(na_class)->unexpected_msg_queue_lock); - HG_QUEUE_PUSH_TAIL(&NA_SM_CLASS(na_class)->unexpected_msg_queue, - na_sm_unexpected_info, entry); - hg_thread_spin_unlock( - &NA_SM_CLASS(na_class)->unexpected_msg_queue_lock); + hg_thread_spin_lock(&unexpected_msg_queue->lock); + HG_QUEUE_PUSH_TAIL( + &unexpected_msg_queue->queue, na_sm_unexpected_info, entry); + hg_thread_spin_unlock(&unexpected_msg_queue->lock); } done: return ret; + +error: + free(na_sm_unexpected_info); + return ret; } /*---------------------------------------------------------------------------*/ static na_return_t -na_sm_progress_expected(na_class_t *na_class, struct na_sm_addr *poll_addr, - na_sm_cacheline_hdr_t na_sm_hdr) +na_sm_process_expected(struct na_sm_op_queue *expected_op_queue, + struct na_sm_addr *poll_addr, na_sm_msg_hdr_t msg_hdr) { struct na_sm_op_id *na_sm_op_id = NULL; na_return_t ret = NA_SUCCESS; - hg_thread_spin_lock( - &NA_SM_CLASS(na_class)->expected_op_queue_lock); - HG_QUEUE_FOREACH(na_sm_op_id, - &NA_SM_CLASS(na_class)->expected_op_queue, entry) { - if (na_sm_op_id->na_sm_addr == poll_addr - && na_sm_op_id->info.msg.tag == na_sm_hdr.hdr.tag) { - HG_QUEUE_REMOVE(&NA_SM_CLASS(na_class)->expected_op_queue, - na_sm_op_id, na_sm_op_id, entry); + NA_LOG_DEBUG("Processing expected msg"); + + /* Try to match addr/tag */ + hg_thread_spin_lock(&expected_op_queue->lock); + HG_QUEUE_FOREACH (na_sm_op_id, &expected_op_queue->queue, entry) { + if (na_sm_op_id->na_sm_addr == poll_addr && + na_sm_op_id->info.msg.tag == msg_hdr.hdr.tag) { + HG_QUEUE_REMOVE( + &expected_op_queue->queue, na_sm_op_id, na_sm_op_id, entry); hg_atomic_and32(&na_sm_op_id->status, ~NA_SM_OP_QUEUED); break; } } - hg_thread_spin_unlock( - &NA_SM_CLASS(na_class)->expected_op_queue_lock); + hg_thread_spin_unlock(&expected_op_queue->lock); - NA_CHECK_ERROR(na_sm_op_id == NULL, done, ret, NA_INVALID_ARG, - "Invalid operation ID"); + NA_CHECK_ERROR( + na_sm_op_id == NULL, done, ret, NA_INVALID_ARG, "Invalid operation ID"); /* Cannot have an already completed operation ID, TODO add sanity check */ - na_sm_op_id->info.msg.actual_buf_size = na_sm_hdr.hdr.buf_size; + na_sm_op_id->info.msg.actual_buf_size = msg_hdr.hdr.buf_size; + + /* Copy buffer */ + na_sm_buf_copy_from(&poll_addr->shared_region->copy_bufs, + msg_hdr.hdr.buf_idx, na_sm_op_id->info.msg.buf.ptr, + msg_hdr.hdr.buf_size); - /* Copy and free buffer atomically */ - na_sm_copy_and_free_buf(na_class, poll_addr->na_sm_copy_buf, - na_sm_op_id->info.msg.buf.ptr, na_sm_hdr.hdr.buf_size, - na_sm_hdr.hdr.buf_idx); + /* Release buffer */ + na_sm_buf_release( + &poll_addr->shared_region->copy_bufs, msg_hdr.hdr.buf_idx); /* Complete operation */ - ret = na_sm_complete(na_sm_op_id); + ret = na_sm_complete(na_sm_op_id, 0); NA_CHECK_NA_ERROR(done, ret, "Could not complete operation"); done: @@ -2286,18 +2882,19 @@ na_sm_progress_expected(na_class_t *na_class, struct na_sm_addr *poll_addr, /*---------------------------------------------------------------------------*/ static na_return_t -na_sm_progress_retries(na_class_t *na_class) +na_sm_process_retries(struct na_sm_op_queue *retry_op_queue) { struct na_sm_op_id *na_sm_op_id = NULL; - unsigned int idx_reserved; + unsigned int buf_idx; na_return_t ret = NA_SUCCESS; do { - na_bool_t canceled = NA_FALSE; + na_sm_msg_hdr_t msg_hdr; + na_bool_t rc; - hg_thread_spin_lock(&NA_SM_CLASS(na_class)->retry_op_queue_lock); - na_sm_op_id = HG_QUEUE_FIRST(&NA_SM_CLASS(na_class)->retry_op_queue); - hg_thread_spin_unlock(&NA_SM_CLASS(na_class)->retry_op_queue_lock); + hg_thread_spin_lock(&retry_op_queue->lock); + na_sm_op_id = HG_QUEUE_FIRST(&retry_op_queue->queue); + hg_thread_spin_unlock(&retry_op_queue->lock); if (!na_sm_op_id) break; @@ -2305,36 +2902,60 @@ na_sm_progress_retries(na_class_t *na_class) NA_LOG_DEBUG("Attempting to retry %p", na_sm_op_id); /* Try to reserve buffer atomically */ - if (na_sm_reserve_and_copy_buf(na_class, - na_sm_op_id->na_sm_addr->na_sm_copy_buf, - na_sm_op_id->info.msg.buf.const_ptr, - na_sm_op_id->info.msg.buf_size, &idx_reserved) == NA_AGAIN) + if (na_sm_buf_reserve( + &na_sm_op_id->na_sm_addr->shared_region->copy_bufs, &buf_idx) == + NA_AGAIN) break; - /* Successfully reserved a buffer */ - hg_thread_spin_lock(&NA_SM_CLASS(na_class)->retry_op_queue_lock); + /* Successfully reserved a buffer, check that the operation has not + * been canceled in the meantime so that we can dequeue the op from + * the queue properly. */ + hg_thread_spin_lock(&retry_op_queue->lock); + if ((hg_atomic_get32(&na_sm_op_id->status) & NA_SM_OP_CANCELED)) { - canceled = NA_TRUE; - na_sm_release_buf(na_sm_op_id->na_sm_addr->na_sm_copy_buf, - idx_reserved); - } else { - HG_QUEUE_REMOVE(&NA_SM_CLASS(na_class)->retry_op_queue, - na_sm_op_id, na_sm_op_id, entry); - hg_atomic_and32(&na_sm_op_id->status, ~NA_SM_OP_QUEUED); + hg_thread_spin_unlock(&retry_op_queue->lock); + na_sm_buf_release( + &na_sm_op_id->na_sm_addr->shared_region->copy_bufs, buf_idx); + continue; } - hg_thread_spin_unlock(&NA_SM_CLASS(na_class)->retry_op_queue_lock); - if (!canceled) { - /* Insert message into ring buffer (complete OP ID) */ - ret = na_sm_msg_insert(na_class, na_sm_op_id, idx_reserved); - NA_CHECK_NA_ERROR(error, ret, "Could not insert message"); + HG_QUEUE_REMOVE( + &retry_op_queue->queue, na_sm_op_id, na_sm_op_id, entry); + hg_atomic_and32(&na_sm_op_id->status, ~NA_SM_OP_QUEUED); + + hg_thread_spin_unlock(&retry_op_queue->lock); + + /* Copy buffer */ + na_sm_buf_copy_to(&na_sm_op_id->na_sm_addr->shared_region->copy_bufs, + buf_idx, na_sm_op_id->info.msg.buf.const_ptr, + na_sm_op_id->info.msg.buf_size); + + /* Post message to queue */ + msg_hdr.hdr.type = na_sm_op_id->completion_data.callback_info.type; + msg_hdr.hdr.buf_idx = buf_idx & 0xff; + msg_hdr.hdr.buf_size = na_sm_op_id->info.msg.buf_size & 0xffff; + msg_hdr.hdr.tag = na_sm_op_id->info.msg.tag; + + rc = na_sm_msg_queue_push(na_sm_op_id->na_sm_addr->tx_queue, msg_hdr); + NA_CHECK_ERROR(rc == NA_FALSE, error, ret, NA_AGAIN, "Full queue"); + + /* Notify remote if notifications are enabled */ + if (na_sm_op_id->na_sm_addr->tx_notify > 0) { + ret = na_sm_event_set(na_sm_op_id->na_sm_addr->tx_notify); + NA_CHECK_NA_ERROR( + error, ret, "Could not send completion notification"); } + + /* Immediate completion, add directly to completion queue. */ + ret = na_sm_complete(na_sm_op_id, 0); + NA_CHECK_NA_ERROR(error, ret, "Could not complete operation"); } while (1); return ret; error: - na_sm_release_buf(na_sm_op_id->na_sm_addr->na_sm_copy_buf, idx_reserved); + na_sm_buf_release( + &na_sm_op_id->na_sm_addr->shared_region->copy_bufs, buf_idx); hg_atomic_decr32(&na_sm_op_id->na_sm_addr->ref_count); hg_atomic_decr32(&na_sm_op_id->ref_count); @@ -2343,33 +2964,32 @@ na_sm_progress_retries(na_class_t *na_class) /*---------------------------------------------------------------------------*/ static na_return_t -na_sm_complete(struct na_sm_op_id *na_sm_op_id) +na_sm_complete(struct na_sm_op_id *na_sm_op_id, int notify) { struct na_cb_info *callback_info = NULL; - na_bool_t canceled = NA_FALSE; na_return_t ret = NA_SUCCESS; + hg_util_int32_t status; /* Mark op id as completed before checking for cancelation */ - if (hg_atomic_or32(&na_sm_op_id->status, NA_SM_OP_COMPLETED) - & NA_SM_OP_CANCELED) { - /* If it was canceled while being processed, set callback ret accordingly */ - NA_LOG_DEBUG("Operation ID %p was canceled", na_sm_op_id); - canceled = NA_TRUE; - } + status = hg_atomic_or32(&na_sm_op_id->status, NA_SM_OP_COMPLETED); /* Init callback info */ callback_info = &na_sm_op_id->completion_data.callback_info; - callback_info->ret = (canceled) ? NA_CANCELED : ret; + + /* Check for current status before completing */ + if (status & NA_SM_OP_CANCELED) { + /* If it was canceled while being processed, set callback ret + * accordingly */ + NA_LOG_DEBUG("Operation ID %p was canceled", na_sm_op_id); + callback_info->ret = NA_CANCELED; + } else + callback_info->ret = NA_SUCCESS; switch (callback_info->type) { - case NA_CB_LOOKUP: - callback_info->info.lookup.addr = - (na_addr_t) na_sm_op_id->na_sm_addr; - break; case NA_CB_SEND_UNEXPECTED: break; case NA_CB_RECV_UNEXPECTED: - if (canceled) { + if (callback_info->ret != NA_SUCCESS) { /* In case of cancellation where no recv'd data */ callback_info->info.recv_unexpected.actual_buf_size = 0; callback_info->info.recv_unexpected.source = NA_ADDR_NULL; @@ -2401,10 +3021,17 @@ na_sm_complete(struct na_sm_op_id *na_sm_op_id) } /* Add OP to NA completion queue */ - ret = na_cb_completion_add(na_sm_op_id->context, - &na_sm_op_id->completion_data); + ret = na_cb_completion_add( + na_sm_op_id->context, &na_sm_op_id->completion_data); NA_CHECK_NA_ERROR(done, ret, "Could not add callback to completion queue"); + /* Notify local completion */ + if (notify > 0) { + int rc = hg_event_set(notify); + NA_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, + na_sm_errno_to_na(errno), "Could not signal completion"); + } + done: return ret; } @@ -2415,8 +3042,8 @@ na_sm_release(void *arg) { struct na_sm_op_id *na_sm_op_id = (struct na_sm_op_id *) arg; - NA_CHECK_WARNING(na_sm_op_id - && (!(hg_atomic_get32(&na_sm_op_id->status) & NA_SM_OP_COMPLETED)), + NA_CHECK_WARNING(na_sm_op_id && (!(hg_atomic_get32(&na_sm_op_id->status) & + NA_SM_OP_COMPLETED)), "Releasing resources from an uncompleted operation"); if (na_sm_op_id->na_sm_addr) { @@ -2446,106 +3073,71 @@ static na_return_t na_sm_initialize(na_class_t *na_class, const struct na_info NA_UNUSED *na_info, na_bool_t listen) { - static hg_atomic_int32_t id = HG_ATOMIC_VAR_INIT(0); - struct na_sm_addr *na_sm_addr = NULL; + static hg_atomic_int32_t sm_id_g = HG_ATOMIC_VAR_INIT(0); pid_t pid; + unsigned int id; char *username = NULL; - hg_poll_set_t *poll_set; - na_bool_t no_wait = NA_FALSE, no_retry = NA_FALSE; - int local_notify; + na_bool_t no_wait = NA_FALSE; + na_uint8_t max_contexts = 1; /* Default */ na_return_t ret = NA_SUCCESS; - /* TODO parse host name */ - /* Get init info */ if (na_info->na_init_info) { /* Progress mode */ if (na_info->na_init_info->progress_mode & NA_NO_BLOCK) no_wait = NA_TRUE; - if (na_info->na_init_info->progress_mode & NA_NO_RETRY) - no_retry = NA_TRUE; + /* Max contexts */ + max_contexts = na_info->na_init_info->max_contexts; } /* Get PID */ pid = getpid(); + /* Generate new SM ID */ + id = (unsigned int) hg_atomic_incr32(&sm_id_g) - 1; + NA_CHECK_ERROR(id > UINT8_MAX, error, ret, NA_OVERFLOW, + "Reached maximum number of SM instances for this process"); + /* Get username */ username = getlogin_safe(); - NA_CHECK_ERROR(username == NULL, done, ret, NA_PROTOCOL_ERROR, + NA_CHECK_ERROR(username == NULL, error, ret, na_sm_errno_to_na(errno), "Could not query login name"); - /* Initialize errno */ + /* Reset errno */ errno = 0; /* Initialize private data */ na_class->plugin_class = malloc(sizeof(struct na_sm_class)); - NA_CHECK_ERROR(na_class->plugin_class == NULL, done, ret, NA_NOMEM, + NA_CHECK_ERROR(na_class->plugin_class == NULL, error, ret, NA_NOMEM, "Could not allocate NA private data class"); memset(na_class->plugin_class, 0, sizeof(struct na_sm_class)); NA_SM_CLASS(na_class)->no_wait = no_wait; - NA_SM_CLASS(na_class)->no_retry = no_retry; + NA_SM_CLASS(na_class)->max_contexts = max_contexts; /* Copy username */ NA_SM_CLASS(na_class)->username = strdup(username); - NA_CHECK_ERROR(NA_SM_CLASS(na_class)->username == NULL, done, ret, NA_NOMEM, - "Could not dup username"); + NA_CHECK_ERROR(NA_SM_CLASS(na_class)->username == NULL, error, ret, + NA_NOMEM, "Could not dup username"); - /* Create poll set to wait for events */ - poll_set = hg_poll_create(); - NA_CHECK_ERROR(poll_set == NULL, done, ret, NA_PROTOCOL_ERROR, - "Cannot create poll set"); - NA_SM_CLASS(na_class)->poll_set = poll_set; + NA_LOG_DEBUG( + "Opening new endpoint for %s with PID=%d, ID=%u", username, pid, id); - /* Create self addr */ - na_sm_addr = (struct na_sm_addr *) malloc(sizeof(struct na_sm_addr)); - NA_CHECK_ERROR(na_sm_addr == NULL, done, ret, NA_NOMEM, - "Could not allocate NA SM addr"); - memset(na_sm_addr, 0, sizeof(struct na_sm_addr)); - na_sm_addr->pid = pid; - na_sm_addr->id = (unsigned int) hg_atomic_incr32(&id) - 1; - na_sm_addr->self = NA_TRUE; - hg_atomic_init32(&na_sm_addr->ref_count, 1); - /* If we're listening, create a new shm region */ - if (listen) { - ret = na_sm_setup_shm(na_class, na_sm_addr); - NA_CHECK_NA_ERROR(done, ret, "Could not setup shm"); - } - /* Create local signal event on self address */ - local_notify = hg_event_create(); - NA_CHECK_ERROR(local_notify == -1, done, ret, NA_PROTOCOL_ERROR, - "hg_event_create() failed"); - na_sm_addr->local_notify = local_notify; + /* Open endpoint */ + ret = na_sm_endpoint_open(&NA_SM_CLASS(na_class)->endpoint, username, pid, + id & 0xff, listen, no_wait); + NA_CHECK_NA_ERROR( + error, ret, "Could not open endpoint for PID=%d, ID=%u", pid, id); - /* Add local notify to poll set */ - ret = na_sm_poll_register(na_class, NA_SM_NOTIFY, na_sm_addr); - NA_CHECK_NA_ERROR(done, ret, "Could not add notify to poll set"); - NA_SM_CLASS(na_class)->self_addr = na_sm_addr; + return ret; - /* Initialize queues */ - HG_QUEUE_INIT(&NA_SM_CLASS(na_class)->accepted_addr_queue); - HG_QUEUE_INIT(&NA_SM_CLASS(na_class)->poll_addr_queue); - HG_QUEUE_INIT(&NA_SM_CLASS(na_class)->unexpected_msg_queue); - HG_QUEUE_INIT(&NA_SM_CLASS(na_class)->lookup_op_queue); - HG_QUEUE_INIT(&NA_SM_CLASS(na_class)->unexpected_op_queue); - HG_QUEUE_INIT(&NA_SM_CLASS(na_class)->expected_op_queue); - HG_QUEUE_INIT(&NA_SM_CLASS(na_class)->retry_op_queue); - - /* Initialize mutexes */ - hg_thread_spin_init(&NA_SM_CLASS(na_class)->accepted_addr_queue_lock); - hg_thread_spin_init(&NA_SM_CLASS(na_class)->poll_addr_queue_lock); - hg_thread_spin_init(&NA_SM_CLASS(na_class)->unexpected_msg_queue_lock); - hg_thread_spin_init(&NA_SM_CLASS(na_class)->lookup_op_queue_lock); - hg_thread_spin_init(&NA_SM_CLASS(na_class)->unexpected_op_queue_lock); - hg_thread_spin_init(&NA_SM_CLASS(na_class)->expected_op_queue_lock); - hg_thread_spin_init(&NA_SM_CLASS(na_class)->retry_op_queue_lock); - hg_thread_spin_init(&NA_SM_CLASS(na_class)->copy_buf_lock); +error: + if (na_class->plugin_class) { + free(NA_SM_CLASS(na_class)->username); + free(na_class->plugin_class); + na_class->plugin_class = NULL; + } -done: return ret; - -//error: -// TODO -// return ret; } /*---------------------------------------------------------------------------*/ @@ -2553,66 +3145,20 @@ static na_return_t na_sm_finalize(na_class_t *na_class) { na_return_t ret = NA_SUCCESS; - na_bool_t empty; - int rc; if (!na_class->plugin_class) goto done; - /* Check that lookup op queue is empty */ - empty = HG_QUEUE_IS_EMPTY(&NA_SM_CLASS(na_class)->lookup_op_queue); - NA_CHECK_ERROR(empty == NA_FALSE, done, ret, NA_PROTOCOL_ERROR, - "Lookup op queue should be empty"); - - /* Check that unexpected op queue is empty */ - empty = HG_QUEUE_IS_EMPTY(&NA_SM_CLASS(na_class)->unexpected_op_queue); - NA_CHECK_ERROR(empty == NA_FALSE, done, ret, NA_PROTOCOL_ERROR, - "Unexpected op queue should be empty"); - - /* Check that unexpected message queue is empty */ - empty = HG_QUEUE_IS_EMPTY(&NA_SM_CLASS(na_class)->unexpected_msg_queue); - NA_CHECK_ERROR(empty == NA_FALSE, done, ret, NA_PROTOCOL_ERROR, - "Unexpected msg queue should be empty"); - - /* Check that expected op queue is empty */ - empty = HG_QUEUE_IS_EMPTY(&NA_SM_CLASS(na_class)->expected_op_queue); - NA_CHECK_ERROR(empty == NA_FALSE, done, ret, NA_PROTOCOL_ERROR, - "Expected op queue should be empty"); - - /* Check that retry op queue is empty */ - empty = HG_QUEUE_IS_EMPTY(&NA_SM_CLASS(na_class)->retry_op_queue); - NA_CHECK_ERROR(empty == NA_FALSE, done, ret, NA_PROTOCOL_ERROR, - "Retry op queue should be empty"); - - /* Check that accepted addr queue is empty */ - while (!HG_QUEUE_IS_EMPTY(&NA_SM_CLASS(na_class)->accepted_addr_queue)) { - struct na_sm_addr *na_sm_addr = HG_QUEUE_FIRST( - &NA_SM_CLASS(na_class)->accepted_addr_queue); - ret = na_sm_addr_free(na_class, na_sm_addr); - NA_CHECK_NA_ERROR(done, ret, "Could not free accepted addr"); - } - - /* Free self addr */ - ret = na_sm_addr_free(na_class, NA_SM_CLASS(na_class)->self_addr); - NA_CHECK_NA_ERROR(done, ret, "Could not free self addr"); + NA_LOG_DEBUG("Closing endpoint for %s", NA_SM_CLASS(na_class)->username); - /* Close poll set */ - rc = hg_poll_destroy(NA_SM_CLASS(na_class)->poll_set); - NA_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, NA_PROTOCOL_ERROR, - "hg_poll_destroy() failed"); - - /* Destroy mutexes */ - hg_thread_spin_destroy(&NA_SM_CLASS(na_class)->accepted_addr_queue_lock); - hg_thread_spin_destroy(&NA_SM_CLASS(na_class)->poll_addr_queue_lock); - hg_thread_spin_destroy(&NA_SM_CLASS(na_class)->unexpected_msg_queue_lock); - hg_thread_spin_destroy(&NA_SM_CLASS(na_class)->lookup_op_queue_lock); - hg_thread_spin_destroy(&NA_SM_CLASS(na_class)->unexpected_op_queue_lock); - hg_thread_spin_destroy(&NA_SM_CLASS(na_class)->expected_op_queue_lock); - hg_thread_spin_destroy(&NA_SM_CLASS(na_class)->retry_op_queue_lock); - hg_thread_spin_destroy(&NA_SM_CLASS(na_class)->copy_buf_lock); + /* Close endpoint */ + ret = na_sm_endpoint_close( + &NA_SM_CLASS(na_class)->endpoint, NA_SM_CLASS(na_class)->username); + NA_CHECK_NA_ERROR(done, ret, "Could not close endpoint"); free(NA_SM_CLASS(na_class)->username); free(na_class->plugin_class); + na_class->plugin_class = NULL; done: return ret; @@ -2626,20 +3172,19 @@ na_sm_cleanup(void) char *username = getlogin_safe(); int ret; - sprintf(pathname, "%s/%s_%s", NA_SM_TMP_DIRECTORY, - NA_SM_SHM_PREFIX, username); + sprintf( + pathname, "%s/%s_%s", NA_SM_TMP_DIRECTORY, NA_SM_SHM_PREFIX, username); /* We need to remove all files first before being able to remove the * directories */ - ret = nftw(pathname, na_sm_cleanup_file, NA_SM_CLEANUP_NFDS, + ret = nftw(pathname, na_sm_sock_path_cleanup, NA_SM_CLEANUP_NFDS, FTW_PHYS | FTW_DEPTH); - NA_CHECK_WARNING(ret != 0 && errno != ENOENT, "nftw() failed (%s)", - strerror(errno)); + NA_CHECK_WARNING( + ret != 0 && errno != ENOENT, "nftw() failed (%s)", strerror(errno)); - ret = nftw(NA_SM_SHM_PATH, na_sm_cleanup_shm, NA_SM_CLEANUP_NFDS, - FTW_PHYS); - NA_CHECK_WARNING(ret != 0 && errno != ENOENT, "nftw() failed (%s)", - strerror(errno)); + ret = nftw(NA_SM_SHM_PATH, na_sm_shm_cleanup, NA_SM_CLEANUP_NFDS, FTW_PHYS); + NA_CHECK_WARNING( + ret != 0 && errno != ENOENT, "nftw() failed (%s)", strerror(errno)); } /*---------------------------------------------------------------------------*/ @@ -2649,9 +3194,10 @@ na_sm_op_create(na_class_t *na_class) struct na_sm_op_id *na_sm_op_id = NULL; na_sm_op_id = (struct na_sm_op_id *) malloc(sizeof(struct na_sm_op_id)); - NA_CHECK_ERROR_NORET(na_sm_op_id == NULL, done, - "Could not allocate NA SM operation ID"); + NA_CHECK_ERROR_NORET( + na_sm_op_id == NULL, done, "Could not allocate NA SM operation ID"); memset(na_sm_op_id, 0, sizeof(struct na_sm_op_id)); + na_sm_op_id->na_class = na_class; hg_atomic_init32(&na_sm_op_id->ref_count, 1); /* Completed by default */ @@ -2670,7 +3216,6 @@ static na_return_t na_sm_op_destroy(na_class_t NA_UNUSED *na_class, na_op_id_t op_id) { struct na_sm_op_id *na_sm_op_id = (struct na_sm_op_id *) op_id; - na_return_t ret = NA_SUCCESS; if (hg_atomic_decr32(&na_sm_op_id->ref_count)) { /* Cannot free yet */ @@ -2679,101 +3224,57 @@ na_sm_op_destroy(na_class_t NA_UNUSED *na_class, na_op_id_t op_id) free(na_sm_op_id); done: - return ret; + return NA_SUCCESS; } /*---------------------------------------------------------------------------*/ static na_return_t -na_sm_addr_lookup(na_class_t *na_class, na_context_t *context, - na_cb_t callback, void *arg, const char *name, na_op_id_t *op_id) +na_sm_addr_lookup(na_class_t *na_class, const char *name, na_addr_t *addr) { - struct na_sm_op_id *na_sm_op_id = NULL; + struct na_sm_endpoint *na_sm_endpoint = &NA_SM_CLASS(na_class)->endpoint; struct na_sm_addr *na_sm_addr = NULL; - struct na_sm_copy_buf *na_sm_copy_buf = NULL; - char filename[NA_SM_MAX_FILENAME]; - char pathname[NA_SM_MAX_FILENAME]; - int conn_sock; - char *name_string = NULL, *short_name = NULL; + pid_t pid; + na_uint8_t id; + na_uint64_t addr_key; na_return_t ret = NA_SUCCESS; - /* Check op_id */ - NA_CHECK_ERROR( - op_id == NULL || op_id == NA_OP_ID_IGNORE || *op_id == NA_OP_ID_NULL, - done, ret, NA_INVALID_ARG, "Invalid operation ID"); + /* Extra info from string */ + ret = na_sm_string_to_addr(name, &pid, &id); + NA_CHECK_NA_ERROR(done, ret, "Could not convert string to address"); - na_sm_op_id = (struct na_sm_op_id *) *op_id; - NA_CHECK_ERROR(!(hg_atomic_get32(&na_sm_op_id->status) & NA_SM_OP_COMPLETED), - done, ret, NA_BUSY, "Attempting to use OP ID that was not completed"); - /* Make sure op ID is fully released before re-using it */ - while (hg_atomic_cas32(&na_sm_op_id->ref_count, 1, 2) != HG_UTIL_TRUE) - cpu_spinwait(); + NA_LOG_DEBUG("Lookup addr for PID=%d, ID=%d", pid, id); - na_sm_op_id->context = context; - na_sm_op_id->completion_data.callback_info.type = NA_CB_LOOKUP; - na_sm_op_id->completion_data.callback = callback; - na_sm_op_id->completion_data.callback_info.arg = arg; - na_sm_op_id->na_sm_addr = NULL; - hg_atomic_set32(&na_sm_op_id->status, 0); + /* Generate key */ + addr_key = na_sm_addr_to_key(pid, id); - /* Allocate addr */ - na_sm_addr = (struct na_sm_addr *) malloc(sizeof(struct na_sm_addr)); - NA_CHECK_ERROR(na_sm_addr == NULL, done, ret, NA_NOMEM, - "Could not allocate NA SM addr"); - memset(na_sm_addr, 0, sizeof(struct na_sm_addr)); - hg_atomic_init32(&na_sm_addr->ref_count, 2); /* Extra refcount */ + /* Lookup addr from hash table */ + na_sm_addr = na_sm_addr_map_lookup(&na_sm_endpoint->addr_map, addr_key); + if (!na_sm_addr) { + struct na_sm_lookup_args args = {.endpoint = na_sm_endpoint, + .username = NA_SM_CLASS(na_class)->username, + .pid = pid, + .id = id}; + na_return_t na_ret; - na_sm_op_id->na_sm_addr = na_sm_addr; + NA_LOG_DEBUG("Addess was not found, attempting to insert it (key=%lu)", + (long unsigned int) addr_key); - /** - * Clean up name, strings can be of the format: - * :// - */ - name_string = strdup(name); - NA_CHECK_ERROR(name_string == NULL, done, ret, NA_NOMEM, - "Could not duplicate string"); + /* Insert new entry and create new address if needed */ + na_ret = na_sm_addr_map_insert(&na_sm_endpoint->addr_map, addr_key, + na_sm_addr_lookup_insert_cb, &args, &na_sm_addr); + NA_CHECK_ERROR(na_ret != NA_SUCCESS && na_ret != NA_EXIST, done, ret, + na_ret, "Could not insert new address"); + } else { + NA_LOG_DEBUG( + "Addess was found (key=%lu)", (long unsigned int) addr_key); + } - if (strstr(name_string, ":") != NULL) { - strtok_r(name_string, ":", &short_name); - short_name += 2; - } else - short_name = name_string; + /* Increment refcount */ + hg_atomic_incr32(&na_sm_addr->ref_count); - /* Get PID / ID from name */ - sscanf(short_name, "%d/%u", &na_sm_addr->pid, &na_sm_addr->id); - - /* Open shared copy buf */ - NA_SM_GEN_SHM_NAME(filename, NA_SM_CLASS(na_class)->username, na_sm_addr); - na_sm_copy_buf = (struct na_sm_copy_buf *) na_sm_open_shared_buf( - filename, sizeof(struct na_sm_copy_buf), NA_FALSE); - NA_CHECK_ERROR(na_sm_copy_buf == NULL, done, ret, NA_PROTOCOL_ERROR, - "Could not open copy buffer"); - na_sm_addr->na_sm_copy_buf = na_sm_copy_buf; - - /* Open SHM sock */ - NA_SM_GEN_SOCK_PATH(pathname, NA_SM_CLASS(na_class)->username, na_sm_addr); - ret = na_sm_create_sock(pathname, NA_FALSE, &conn_sock); - NA_CHECK_NA_ERROR(done, ret, "Could not create sock"); - na_sm_addr->sock = conn_sock; - /* We only need to receive conn ID in sock progress */ - na_sm_addr->sock_progress = NA_SM_CONN_ID; - - /* Push op ID to lookup op queue */ - hg_thread_spin_lock(&NA_SM_CLASS(na_class)->lookup_op_queue_lock); - HG_QUEUE_PUSH_TAIL(&NA_SM_CLASS(na_class)->lookup_op_queue, na_sm_op_id, - entry); - hg_thread_spin_unlock(&NA_SM_CLASS(na_class)->lookup_op_queue_lock); - - /* Add conn_sock to poll set */ - ret = na_sm_poll_register(na_class, NA_SM_SOCK, na_sm_addr); - NA_CHECK_NA_ERROR(done, ret, "Could not add conn_sock to poll set"); - - /* Send addr info (PID / ID) */ - ret = na_sm_send_addr_info(na_class, na_sm_addr); - NA_CHECK_NA_ERROR(done, ret, "Could not send addr info"); + *addr = (na_addr_t) na_sm_addr; done: - free(name_string); - return ret; } @@ -2781,148 +3282,28 @@ na_sm_addr_lookup(na_class_t *na_class, na_context_t *context, static na_return_t na_sm_addr_free(na_class_t *na_class, na_addr_t addr) { + struct na_sm_endpoint *na_sm_endpoint = &NA_SM_CLASS(na_class)->endpoint; struct na_sm_addr *na_sm_addr = (struct na_sm_addr *) addr; - const char *copy_buf_name = NULL, *send_ring_buf_name = NULL, - *recv_ring_buf_name = NULL, *pathname = NULL; - char na_sm_copy_buf_name[NA_SM_MAX_FILENAME], - na_sm_send_ring_buf_name[NA_SM_MAX_FILENAME], - na_sm_recv_ring_buf_name[NA_SM_MAX_FILENAME], - na_sock_name[NA_SM_MAX_FILENAME]; na_return_t ret = NA_SUCCESS; - int rc; - NA_CHECK_ERROR(na_sm_addr == NULL, done, ret, NA_INVALID_ARG, - "NULL SM addr"); + if (!na_sm_addr) + goto done; if (hg_atomic_decr32(&na_sm_addr->ref_count)) /* Cannot free yet */ goto done; - if (na_sm_addr->accepted) { /* Created by accept */ - hg_thread_spin_lock(&NA_SM_CLASS(na_class)->accepted_addr_queue_lock); - /* Remove the addr from accepted addr queue */ - HG_QUEUE_REMOVE(&NA_SM_CLASS(na_class)->accepted_addr_queue, na_sm_addr, - na_sm_addr, entry); - hg_thread_spin_unlock(&NA_SM_CLASS(na_class)->accepted_addr_queue_lock); - } - - /* Deregister event file descriptors from poll set */ - ret = na_sm_poll_deregister(na_class, NA_SM_NOTIFY, na_sm_addr); - NA_CHECK_NA_ERROR(done, ret, "Could not delete notify from poll set"); - - /* Destroy local event */ -#ifdef HG_UTIL_HAS_SYSEVENTFD_H - rc = hg_event_destroy(na_sm_addr->local_notify); - NA_CHECK_ERROR(rc == HG_UTIL_FAIL, done, ret, NA_PROTOCOL_ERROR, - "hg_event_destroy() failed"); -#endif - - // TODO cleanup - if (!na_sm_addr->self) { /* Created by lookup/connect or accept */ -#ifndef HG_UTIL_HAS_SYSEVENTFD_H - char na_sm_local_event_name[NA_SM_MAX_FILENAME], - na_sm_remote_event_name[NA_SM_MAX_FILENAME]; - const char *local_event_name = NULL, *remote_event_name = NULL; -#endif - - /* Deregister sock file descriptor */ - ret = na_sm_poll_deregister(na_class, NA_SM_SOCK, na_sm_addr); - NA_CHECK_NA_ERROR(done, ret, "Could not delete sock from poll set"); - - /* Remove addr from poll addr queue */ - hg_thread_spin_lock(&NA_SM_CLASS(na_class)->poll_addr_queue_lock); - HG_QUEUE_REMOVE(&NA_SM_CLASS(na_class)->poll_addr_queue, na_sm_addr, - na_sm_addr, poll_entry); - hg_thread_spin_unlock(&NA_SM_CLASS(na_class)->poll_addr_queue_lock); - - if (na_sm_addr->accepted) { /* Created by accept */ - /* Get file names from ring bufs / events to delete files */ - sprintf(na_sm_send_ring_buf_name, "%s_%s-%d-%d-%d-" NA_SM_SEND_NAME, - NA_SM_SHM_PREFIX, NA_SM_CLASS(na_class)->username, - NA_SM_CLASS(na_class)->self_addr->pid, - NA_SM_CLASS(na_class)->self_addr->id, - na_sm_addr->conn_id); - sprintf(na_sm_recv_ring_buf_name, "%s_%s-%d-%d-%d-" NA_SM_RECV_NAME, - NA_SM_SHM_PREFIX, NA_SM_CLASS(na_class)->username, - NA_SM_CLASS(na_class)->self_addr->pid, - NA_SM_CLASS(na_class)->self_addr->id, - na_sm_addr->conn_id); - send_ring_buf_name = na_sm_send_ring_buf_name; - recv_ring_buf_name = na_sm_recv_ring_buf_name; - -#ifndef HG_UTIL_HAS_SYSEVENTFD_H - sprintf(na_sm_local_event_name, "%s/%s_%s/%d/%u/fifo-%u-%s", - NA_SM_TMP_DIRECTORY, NA_SM_SHM_PREFIX, - NA_SM_CLASS(na_class)->username, - NA_SM_CLASS(na_class)->self_addr->pid, - NA_SM_CLASS(na_class)->self_addr->id, - na_sm_addr->conn_id, NA_SM_RECV_NAME); - sprintf(na_sm_remote_event_name, "%s/%s_%s/%d/%u/fifo-%u-%s", - NA_SM_TMP_DIRECTORY, NA_SM_SHM_PREFIX, - NA_SM_CLASS(na_class)->username, - NA_SM_CLASS(na_class)->self_addr->pid, - NA_SM_CLASS(na_class)->self_addr->id, - na_sm_addr->conn_id, NA_SM_SEND_NAME); - local_event_name = na_sm_local_event_name; - remote_event_name = na_sm_remote_event_name; -#endif - } - - /* Destroy events */ -#ifdef HG_UTIL_HAS_SYSEVENTFD_H - rc = hg_event_destroy(na_sm_addr->remote_notify); - NA_CHECK_ERROR(rc == HG_UTIL_FAIL, done, ret, NA_PROTOCOL_ERROR, - "hg_event_destroy() failed"); -#else - ret = na_sm_event_destroy(local_event_name, na_sm_addr->local_notify); - NA_CHECK_NA_ERROR(done, ret, "na_sm_event_destroy() failed"); - - ret = na_sm_event_destroy(remote_event_name, na_sm_addr->remote_notify); - NA_CHECK_NA_ERROR(done, ret, "na_sm_event_destroy() failed"); -#endif - } else { -#ifndef HG_UTIL_HAS_SYSEVENTFD_H - /* Destroy local event */ - rc = hg_event_destroy(na_sm_addr->local_notify); - NA_CHECK_ERROR(rc == HG_UTIL_FAIL, done, ret, NA_PROTOCOL_ERROR, - "hg_event_destroy() failed"); -#endif - if (na_sm_addr->na_sm_copy_buf) { /* Self addr and listen */ - ret = na_sm_poll_deregister(na_class, NA_SM_ACCEPT, na_sm_addr); - NA_CHECK_NA_ERROR(done, ret, - "Could not delete listen from poll set"); - - NA_SM_GEN_SHM_NAME(na_sm_copy_buf_name, - NA_SM_CLASS(na_class)->username, na_sm_addr); - copy_buf_name = na_sm_copy_buf_name; - NA_SM_GEN_SOCK_PATH(na_sock_name, - NA_SM_CLASS(na_class)->username, na_sm_addr); - pathname = na_sock_name; - } - } - - /* Close sock (delete also tmp dir if pathname is set) */ - ret = na_sm_close_sock(na_sm_addr->sock, pathname); - NA_CHECK_NA_ERROR(done, ret, "Could not close sock"); - - /* Close ring buf (send) */ - ret = na_sm_close_shared_buf(send_ring_buf_name, - na_sm_addr->na_sm_send_ring_buf, sizeof(struct na_sm_ring_buf)); - NA_CHECK_NA_ERROR(done, ret, "Could not close send ring buffer"); - - /* Close ring buf (recv) */ - ret = na_sm_close_shared_buf(recv_ring_buf_name, - na_sm_addr->na_sm_recv_ring_buf, sizeof(struct na_sm_ring_buf)); - NA_CHECK_NA_ERROR(done, ret, "Could not close recv ring buffer"); + NA_LOG_DEBUG( + "Freeing addr for PID=%d, ID=%d", na_sm_addr->pid, na_sm_addr->id); - /* Close copy buf */ - if (!na_sm_addr->accepted) { /* Created by accept */ - ret = na_sm_close_shared_buf(copy_buf_name, na_sm_addr->na_sm_copy_buf, - sizeof(struct na_sm_copy_buf)); - NA_CHECK_NA_ERROR(done, ret, "Could not close copy buffer"); - } + /* Remove address from list of addresses to poll */ + hg_thread_spin_lock(&na_sm_endpoint->poll_addr_list.lock); + HG_LIST_REMOVE(na_sm_addr, entry); + hg_thread_spin_unlock(&na_sm_endpoint->poll_addr_list.lock); - free(na_sm_addr); + ret = na_sm_addr_destroy( + na_sm_endpoint, NA_SM_CLASS(na_class)->username, na_sm_addr); + NA_CHECK_NA_ERROR(done, ret, "Could not destroy address"); done: return ret; @@ -2932,7 +3313,7 @@ na_sm_addr_free(na_class_t *na_class, na_addr_t addr) static na_return_t na_sm_addr_self(na_class_t *na_class, na_addr_t *addr) { - struct na_sm_addr *na_sm_addr = NA_SM_CLASS(na_class)->self_addr; + struct na_sm_addr *na_sm_addr = NA_SM_CLASS(na_class)->endpoint.source_addr; na_return_t ret = NA_SUCCESS; /* Increment refcount */ @@ -2945,8 +3326,8 @@ na_sm_addr_self(na_class_t *na_class, na_addr_t *addr) /*---------------------------------------------------------------------------*/ static na_return_t -na_sm_addr_dup(na_class_t NA_UNUSED *na_class, na_addr_t addr, - na_addr_t *new_addr) +na_sm_addr_dup( + na_class_t NA_UNUSED *na_class, na_addr_t addr, na_addr_t *new_addr) { struct na_sm_addr *na_sm_addr = (struct na_sm_addr *) addr; na_return_t ret = NA_SUCCESS; @@ -2966,17 +3347,16 @@ na_sm_addr_cmp(na_class_t NA_UNUSED *na_class, na_addr_t addr1, na_addr_t addr2) struct na_sm_addr *na_sm_addr1 = (struct na_sm_addr *) addr1; struct na_sm_addr *na_sm_addr2 = (struct na_sm_addr *) addr2; - return (na_sm_addr1->pid == na_sm_addr2->pid) - && (na_sm_addr1->id == na_sm_addr2->id); + return (na_sm_addr1->pid == na_sm_addr2->pid) && + (na_sm_addr1->id == na_sm_addr2->id); } /*---------------------------------------------------------------------------*/ static NA_INLINE na_bool_t -na_sm_addr_is_self(na_class_t NA_UNUSED *na_class, na_addr_t addr) +na_sm_addr_is_self(na_class_t *na_class, na_addr_t addr) { - struct na_sm_addr *na_sm_addr = (struct na_sm_addr *) addr; - - return na_sm_addr->self; + return na_sm_addr_cmp(na_class, + (na_addr_t) NA_SM_CLASS(na_class)->endpoint.source_addr, addr); } /*---------------------------------------------------------------------------*/ @@ -2986,10 +3366,10 @@ na_sm_addr_to_string(na_class_t NA_UNUSED *na_class, char *buf, { struct na_sm_addr *na_sm_addr = (struct na_sm_addr *) addr; na_size_t string_len; - char addr_string[NA_SM_MAX_FILENAME]; + char addr_string[NA_SM_MAX_FILENAME] = {'\0'}; na_return_t ret = NA_SUCCESS; - sprintf(addr_string, "sm://%d/%u", na_sm_addr->pid, na_sm_addr->id); + sprintf(addr_string, "sm://%d/%" SCNu8, na_sm_addr->pid, na_sm_addr->id); string_len = strlen(addr_string); if (buf) { @@ -3003,6 +3383,80 @@ na_sm_addr_to_string(na_class_t NA_UNUSED *na_class, char *buf, return ret; } +/*---------------------------------------------------------------------------*/ +static NA_INLINE na_size_t +na_sm_addr_get_serialize_size(na_class_t NA_UNUSED *na_class, na_addr_t addr) +{ + struct na_sm_addr *na_sm_addr = (struct na_sm_addr *) addr; + + return sizeof(na_sm_addr->pid) + sizeof(na_sm_addr->id); +} + +/*---------------------------------------------------------------------------*/ +static na_return_t +na_sm_addr_serialize(na_class_t NA_UNUSED *na_class, void *buf, + na_size_t buf_size, na_addr_t addr) +{ + struct na_sm_addr *na_sm_addr = (struct na_sm_addr *) addr; + na_uint8_t *p = buf; + na_size_t len = sizeof(na_sm_addr->pid) + sizeof(na_sm_addr->id); + na_return_t ret = NA_SUCCESS; + + NA_CHECK_ERROR(buf_size < len, done, ret, NA_OVERFLOW, + "Buffer size too small for serializing address"); + + /* Encode PID */ + memcpy(p, &na_sm_addr->pid, sizeof(na_sm_addr->pid)); + p += sizeof(na_sm_addr->pid); + + /* Encode ID */ + memcpy(p, &na_sm_addr->id, sizeof(na_sm_addr->id)); + +done: + return ret; +} + +/*---------------------------------------------------------------------------*/ +static na_return_t +na_sm_addr_deserialize( + na_class_t *na_class, na_addr_t *addr, const void *buf, na_size_t buf_size) +{ + struct na_sm_addr *na_sm_addr = NULL; + const na_uint8_t *p = buf; + pid_t pid; + na_uint8_t id; + na_size_t len = sizeof(pid) + sizeof(id); + na_uint64_t addr_key; + na_return_t ret = NA_SUCCESS; + + NA_CHECK_ERROR(buf_size < len, done, ret, NA_OVERFLOW, + "Buffer size too small for serializing address"); + + /* Decode PID */ + memcpy(&pid, p, sizeof(pid)); + p += sizeof(pid); + + /* Decode ID */ + memcpy(&id, p, sizeof(id)); + + /* Generate key */ + addr_key = na_sm_addr_to_key(pid, id); + + /* Lookup addr from hash table */ + na_sm_addr = na_sm_addr_map_lookup( + &NA_SM_CLASS(na_class)->endpoint.addr_map, addr_key); + NA_CHECK_ERROR( + na_sm_addr == NULL, done, ret, NA_NOENTRY, "Could not find address"); + + /* Increment refcount */ + hg_atomic_incr32(&na_sm_addr->ref_count); + + *addr = na_sm_addr; + +done: + return ret; +} + /*---------------------------------------------------------------------------*/ static NA_INLINE na_size_t na_sm_msg_get_max_unexpected_size(const na_class_t NA_UNUSED *na_class) @@ -3033,7 +3487,7 @@ na_sm_msg_send_unexpected(na_class_t *na_class, na_context_t *context, { struct na_sm_op_id *na_sm_op_id = NULL; struct na_sm_addr *na_sm_addr = (struct na_sm_addr *) dest_addr; - unsigned int idx_reserved; + unsigned int buf_idx; na_bool_t reserved = NA_FALSE; na_return_t ret = NA_SUCCESS; @@ -3046,8 +3500,9 @@ na_sm_msg_send_unexpected(na_class_t *na_class, na_context_t *context, done, ret, NA_INVALID_ARG, "Invalid operation ID"); na_sm_op_id = (struct na_sm_op_id *) *op_id; - NA_CHECK_ERROR(!(hg_atomic_get32(&na_sm_op_id->status) & NA_SM_OP_COMPLETED), - done, ret, NA_BUSY, "Attempting to use OP ID that was not completed"); + NA_CHECK_ERROR( + !(hg_atomic_get32(&na_sm_op_id->status) & NA_SM_OP_COMPLETED), done, + ret, NA_BUSY, "Attempting to use OP ID that was not completed"); /* Make sure op ID is fully released before re-using it */ while (hg_atomic_cas32(&na_sm_op_id->ref_count, 1, 2) != HG_UTIL_TRUE) cpu_spinwait(); @@ -3059,38 +3514,59 @@ na_sm_msg_send_unexpected(na_class_t *na_class, na_context_t *context, hg_atomic_incr32(&na_sm_addr->ref_count); na_sm_op_id->na_sm_addr = na_sm_addr; hg_atomic_set32(&na_sm_op_id->status, 0); - /* TODO we assume that buf remains valid (safe because we pre-allocate buffers) */ + /* TODO we assume that buf remains valid (safe because we pre-allocate + * buffers) */ na_sm_op_id->info.msg.buf.const_ptr = buf; na_sm_op_id->info.msg.buf_size = buf_size; na_sm_op_id->info.msg.actual_buf_size = buf_size; na_sm_op_id->info.msg.tag = tag; /* Try to reserve buffer atomically */ - ret = na_sm_reserve_and_copy_buf(na_class, na_sm_addr->na_sm_copy_buf, - buf, buf_size, &idx_reserved); + ret = na_sm_buf_reserve(&na_sm_addr->shared_region->copy_bufs, &buf_idx); if (unlikely(ret == NA_AGAIN)) { - if (NA_SM_CLASS(na_class)->no_retry) - /* Do not attempt to retry */ - NA_GOTO_DONE(error, ret, NA_AGAIN); - else { - NA_LOG_DEBUG("Pushing %p for retry", na_sm_op_id); - - /* Push op ID to retry queue */ - hg_thread_spin_lock(&NA_SM_CLASS(na_class)->retry_op_queue_lock); - HG_QUEUE_PUSH_TAIL(&NA_SM_CLASS(na_class)->retry_op_queue, - na_sm_op_id, entry); - hg_atomic_or32(&na_sm_op_id->status, NA_SM_OP_QUEUED); - hg_thread_spin_unlock(&NA_SM_CLASS(na_class)->retry_op_queue_lock); - - ret = NA_SUCCESS; - } + struct na_sm_op_queue *retry_op_queue = + &NA_SM_CLASS(na_class)->endpoint.retry_op_queue; + + NA_LOG_DEBUG("Pushing %p for retry", na_sm_op_id); + + /* Push op ID to retry queue */ + hg_thread_spin_lock(&retry_op_queue->lock); + HG_QUEUE_PUSH_TAIL(&retry_op_queue->queue, na_sm_op_id, entry); + hg_atomic_or32(&na_sm_op_id->status, NA_SM_OP_QUEUED); + hg_thread_spin_unlock(&retry_op_queue->lock); + + ret = NA_SUCCESS; } else { + na_sm_msg_hdr_t msg_hdr; + na_bool_t rc; + /* Successfully reserved a buffer */ reserved = NA_TRUE; - /* Insert message into ring buffer (complete OP ID) */ - ret = na_sm_msg_insert(na_class, na_sm_op_id, idx_reserved); - NA_CHECK_NA_ERROR(error, ret, "Could not insert message"); + /* Reservation succeeded, copy buffer */ + na_sm_buf_copy_to( + &na_sm_addr->shared_region->copy_bufs, buf_idx, buf, buf_size); + + /* Post message to queue */ + msg_hdr.hdr.type = na_sm_op_id->completion_data.callback_info.type; + msg_hdr.hdr.buf_idx = buf_idx & 0xff; + msg_hdr.hdr.buf_size = buf_size & 0xffff; + msg_hdr.hdr.tag = tag; + + rc = na_sm_msg_queue_push(na_sm_addr->tx_queue, msg_hdr); + NA_CHECK_ERROR(rc == NA_FALSE, error, ret, NA_AGAIN, "Full queue"); + + /* Notify remote if notifications are enabled */ + if (na_sm_addr->tx_notify > 0) { + ret = na_sm_event_set(na_sm_addr->tx_notify); + NA_CHECK_NA_ERROR( + error, ret, "Could not send completion notification"); + } + + /* Immediate completion, add directly to completion queue. */ + ret = na_sm_complete(na_sm_op_id, + NA_SM_CLASS(na_class)->endpoint.source_addr->tx_notify); + NA_CHECK_NA_ERROR(error, ret, "Could not complete operation"); } done: @@ -3098,9 +3574,8 @@ na_sm_msg_send_unexpected(na_class_t *na_class, na_context_t *context, error: if (reserved) - na_sm_release_buf(na_sm_op_id->na_sm_addr->na_sm_copy_buf, - idx_reserved); - hg_atomic_decr32(&na_sm_op_id->na_sm_addr->ref_count); + na_sm_buf_release(&na_sm_addr->shared_region->copy_bufs, buf_idx); + hg_atomic_decr32(&na_sm_addr->ref_count); hg_atomic_decr32(&na_sm_op_id->ref_count); return ret; @@ -3112,6 +3587,8 @@ na_sm_msg_recv_unexpected(na_class_t *na_class, na_context_t *context, na_cb_t callback, void *arg, void *buf, na_size_t buf_size, void NA_UNUSED *plugin_data, na_op_id_t *op_id) { + struct na_sm_unexpected_msg_queue *unexpected_msg_queue = + &NA_SM_CLASS(na_class)->endpoint.unexpected_msg_queue; struct na_sm_unexpected_info *na_sm_unexpected_info; struct na_sm_op_id *na_sm_op_id = NULL; na_return_t ret = NA_SUCCESS; @@ -3125,8 +3602,9 @@ na_sm_msg_recv_unexpected(na_class_t *na_class, na_context_t *context, done, ret, NA_INVALID_ARG, "Invalid operation ID"); na_sm_op_id = (struct na_sm_op_id *) *op_id; - NA_CHECK_ERROR(!(hg_atomic_get32(&na_sm_op_id->status) & NA_SM_OP_COMPLETED), - done, ret, NA_BUSY, "Attempting to use OP ID that was not completed"); + NA_CHECK_ERROR( + !(hg_atomic_get32(&na_sm_op_id->status) & NA_SM_OP_COMPLETED), done, + ret, NA_BUSY, "Attempting to use OP ID that was not completed"); /* Make sure op ID is fully released before re-using it */ while (hg_atomic_cas32(&na_sm_op_id->ref_count, 1, 2) != HG_UTIL_TRUE) cpu_spinwait(); @@ -3141,11 +3619,10 @@ na_sm_msg_recv_unexpected(na_class_t *na_class, na_context_t *context, na_sm_op_id->info.msg.buf_size = buf_size; /* Look for an unexpected message already received */ - hg_thread_spin_lock(&NA_SM_CLASS(na_class)->unexpected_msg_queue_lock); - na_sm_unexpected_info = HG_QUEUE_FIRST( - &NA_SM_CLASS(na_class)->unexpected_msg_queue); - HG_QUEUE_POP_HEAD(&NA_SM_CLASS(na_class)->unexpected_msg_queue, entry); - hg_thread_spin_unlock(&NA_SM_CLASS(na_class)->unexpected_msg_queue_lock); + hg_thread_spin_lock(&unexpected_msg_queue->lock); + na_sm_unexpected_info = HG_QUEUE_FIRST(&unexpected_msg_queue->queue); + HG_QUEUE_POP_HEAD(&unexpected_msg_queue->queue, entry); + hg_thread_spin_unlock(&unexpected_msg_queue->lock); if (unlikely(na_sm_unexpected_info)) { na_sm_op_id->na_sm_addr = na_sm_unexpected_info->na_sm_addr; hg_atomic_incr32(&na_sm_op_id->na_sm_addr->ref_count); @@ -3159,18 +3636,21 @@ na_sm_msg_recv_unexpected(na_class_t *na_class, na_context_t *context, free(na_sm_unexpected_info->buf); free(na_sm_unexpected_info); - ret = na_sm_complete(na_sm_op_id); + ret = na_sm_complete(na_sm_op_id, + NA_SM_CLASS(na_class)->endpoint.source_addr->tx_notify); NA_CHECK_NA_ERROR(error, ret, "Could not complete operation"); } else { + struct na_sm_op_queue *unexpected_op_queue = + &NA_SM_CLASS(na_class)->endpoint.unexpected_op_queue; + na_sm_op_id->info.msg.actual_buf_size = 0; na_sm_op_id->info.msg.tag = 0; /* Nothing has been received yet so add op_id to progress queue */ - hg_thread_spin_lock(&NA_SM_CLASS(na_class)->unexpected_op_queue_lock); - HG_QUEUE_PUSH_TAIL(&NA_SM_CLASS(na_class)->unexpected_op_queue, - na_sm_op_id, entry); + hg_thread_spin_lock(&unexpected_op_queue->lock); + HG_QUEUE_PUSH_TAIL(&unexpected_op_queue->queue, na_sm_op_id, entry); hg_atomic_or32(&na_sm_op_id->status, NA_SM_OP_QUEUED); - hg_thread_spin_unlock(&NA_SM_CLASS(na_class)->unexpected_op_queue_lock); + hg_thread_spin_unlock(&unexpected_op_queue->lock); } done: @@ -3185,14 +3665,14 @@ na_sm_msg_recv_unexpected(na_class_t *na_class, na_context_t *context, /*---------------------------------------------------------------------------*/ static na_return_t -na_sm_msg_send_expected(na_class_t NA_UNUSED *na_class, na_context_t *context, +na_sm_msg_send_expected(na_class_t *na_class, na_context_t *context, na_cb_t callback, void *arg, const void *buf, na_size_t buf_size, void NA_UNUSED *plugin_data, na_addr_t dest_addr, na_uint8_t NA_UNUSED dest_id, na_tag_t tag, na_op_id_t *op_id) { struct na_sm_op_id *na_sm_op_id = NULL; struct na_sm_addr *na_sm_addr = (struct na_sm_addr *) dest_addr; - unsigned int idx_reserved; + unsigned int buf_idx; na_bool_t reserved = NA_FALSE; na_return_t ret = NA_SUCCESS; @@ -3205,8 +3685,9 @@ na_sm_msg_send_expected(na_class_t NA_UNUSED *na_class, na_context_t *context, done, ret, NA_INVALID_ARG, "Invalid operation ID"); na_sm_op_id = (struct na_sm_op_id *) *op_id; - NA_CHECK_ERROR(!(hg_atomic_get32(&na_sm_op_id->status) & NA_SM_OP_COMPLETED), - done, ret, NA_BUSY, "Attempting to use OP ID that was not completed"); + NA_CHECK_ERROR( + !(hg_atomic_get32(&na_sm_op_id->status) & NA_SM_OP_COMPLETED), done, + ret, NA_BUSY, "Attempting to use OP ID that was not completed"); /* Make sure op ID is fully released before re-using it */ while (hg_atomic_cas32(&na_sm_op_id->ref_count, 1, 2) != HG_UTIL_TRUE) cpu_spinwait(); @@ -3218,38 +3699,59 @@ na_sm_msg_send_expected(na_class_t NA_UNUSED *na_class, na_context_t *context, hg_atomic_incr32(&na_sm_addr->ref_count); na_sm_op_id->na_sm_addr = na_sm_addr; hg_atomic_set32(&na_sm_op_id->status, 0); - /* TODO we assume that buf remains valid (safe because we pre-allocate buffers) */ + /* TODO we assume that buf remains valid (safe because we pre-allocate + * buffers) */ na_sm_op_id->info.msg.buf.const_ptr = buf; na_sm_op_id->info.msg.buf_size = buf_size; na_sm_op_id->info.msg.actual_buf_size = buf_size; na_sm_op_id->info.msg.tag = tag; /* Try to reserve buffer atomically */ - ret = na_sm_reserve_and_copy_buf(na_class, na_sm_addr->na_sm_copy_buf, - buf, buf_size, &idx_reserved); + ret = na_sm_buf_reserve(&na_sm_addr->shared_region->copy_bufs, &buf_idx); if (unlikely(ret == NA_AGAIN)) { - if (NA_SM_CLASS(na_class)->no_retry) - /* Do not attempt to retry */ - NA_GOTO_DONE(error, ret, NA_AGAIN); - else { - NA_LOG_DEBUG("Pushing %p for retry", na_sm_op_id); - - /* Push op ID to retry queue */ - hg_thread_spin_lock(&NA_SM_CLASS(na_class)->retry_op_queue_lock); - HG_QUEUE_PUSH_TAIL(&NA_SM_CLASS(na_class)->retry_op_queue, - na_sm_op_id, entry); - hg_atomic_or32(&na_sm_op_id->status, NA_SM_OP_QUEUED); - hg_thread_spin_unlock(&NA_SM_CLASS(na_class)->retry_op_queue_lock); - - ret = NA_SUCCESS; - } + struct na_sm_op_queue *retry_op_queue = + &NA_SM_CLASS(na_class)->endpoint.retry_op_queue; + + NA_LOG_DEBUG("Pushing %p for retry", na_sm_op_id); + + /* Push op ID to retry queue */ + hg_thread_spin_lock(&retry_op_queue->lock); + HG_QUEUE_PUSH_TAIL(&retry_op_queue->queue, na_sm_op_id, entry); + hg_atomic_or32(&na_sm_op_id->status, NA_SM_OP_QUEUED); + hg_thread_spin_unlock(&retry_op_queue->lock); + + ret = NA_SUCCESS; } else { + na_sm_msg_hdr_t msg_hdr; + na_bool_t rc; + /* Successfully reserved a buffer */ reserved = NA_TRUE; - /* Insert message into ring buffer (complete OP ID) */ - ret = na_sm_msg_insert(na_class, na_sm_op_id, idx_reserved); - NA_CHECK_NA_ERROR(error, ret, "Could not insert message"); + /* Reservation succeeded, copy buffer */ + na_sm_buf_copy_to( + &na_sm_addr->shared_region->copy_bufs, buf_idx, buf, buf_size); + + /* Post message to queue */ + msg_hdr.hdr.type = na_sm_op_id->completion_data.callback_info.type; + msg_hdr.hdr.buf_idx = buf_idx & 0xff; + msg_hdr.hdr.buf_size = buf_size & 0xffff; + msg_hdr.hdr.tag = tag; + + rc = na_sm_msg_queue_push(na_sm_addr->tx_queue, msg_hdr); + NA_CHECK_ERROR(rc == NA_FALSE, error, ret, NA_AGAIN, "Full queue"); + + /* Notify remote if notifications are enabled */ + if (na_sm_addr->tx_notify > 0) { + ret = na_sm_event_set(na_sm_addr->tx_notify); + NA_CHECK_NA_ERROR( + error, ret, "Could not send completion notification"); + } + + /* Immediate completion, add directly to completion queue. */ + ret = na_sm_complete(na_sm_op_id, + NA_SM_CLASS(na_class)->endpoint.source_addr->tx_notify); + NA_CHECK_NA_ERROR(error, ret, "Could not complete operation"); } done: @@ -3257,8 +3759,7 @@ na_sm_msg_send_expected(na_class_t NA_UNUSED *na_class, na_context_t *context, error: if (reserved) - na_sm_release_buf(na_sm_op_id->na_sm_addr->na_sm_copy_buf, - idx_reserved); + na_sm_buf_release(&na_sm_addr->shared_region->copy_bufs, buf_idx); hg_atomic_decr32(&na_sm_op_id->na_sm_addr->ref_count); hg_atomic_decr32(&na_sm_op_id->ref_count); @@ -3272,6 +3773,8 @@ na_sm_msg_recv_expected(na_class_t *na_class, na_context_t *context, void NA_UNUSED *plugin_data, na_addr_t source_addr, na_uint8_t NA_UNUSED source_id, na_tag_t tag, na_op_id_t *op_id) { + struct na_sm_op_queue *expected_op_queue = + &NA_SM_CLASS(na_class)->endpoint.expected_op_queue; struct na_sm_op_id *na_sm_op_id = NULL; struct na_sm_addr *na_sm_addr = (struct na_sm_addr *) source_addr; na_return_t ret = NA_SUCCESS; @@ -3285,8 +3788,9 @@ na_sm_msg_recv_expected(na_class_t *na_class, na_context_t *context, done, ret, NA_INVALID_ARG, "Invalid operation ID"); na_sm_op_id = (struct na_sm_op_id *) *op_id; - NA_CHECK_ERROR(!(hg_atomic_get32(&na_sm_op_id->status) & NA_SM_OP_COMPLETED), - done, ret, NA_BUSY, "Attempting to use OP ID that was not completed"); + NA_CHECK_ERROR( + !(hg_atomic_get32(&na_sm_op_id->status) & NA_SM_OP_COMPLETED), done, + ret, NA_BUSY, "Attempting to use OP ID that was not completed"); /* Make sure op ID is fully released before re-using it */ while (hg_atomic_cas32(&na_sm_op_id->ref_count, 1, 2) != HG_UTIL_TRUE) cpu_spinwait(); @@ -3306,11 +3810,10 @@ na_sm_msg_recv_expected(na_class_t *na_class, na_context_t *context, /* Expected messages must always be pre-posted, therefore a message should * never arrive before that call returns (not completes), simply add * op_id to queue */ - hg_thread_spin_lock(&NA_SM_CLASS(na_class)->expected_op_queue_lock); - HG_QUEUE_PUSH_TAIL(&NA_SM_CLASS(na_class)->expected_op_queue, na_sm_op_id, - entry); + hg_thread_spin_lock(&expected_op_queue->lock); + HG_QUEUE_PUSH_TAIL(&expected_op_queue->queue, na_sm_op_id, entry); hg_atomic_or32(&na_sm_op_id->status, NA_SM_OP_QUEUED); - hg_thread_spin_unlock(&NA_SM_CLASS(na_class)->expected_op_queue_lock); + hg_thread_spin_unlock(&expected_op_queue->lock); done: return ret; @@ -3324,8 +3827,8 @@ na_sm_mem_handle_create(na_class_t NA_UNUSED *na_class, void *buf, struct na_sm_mem_handle *na_sm_mem_handle = NULL; na_return_t ret = NA_SUCCESS; - na_sm_mem_handle = (struct na_sm_mem_handle *) malloc( - sizeof(struct na_sm_mem_handle)); + na_sm_mem_handle = + (struct na_sm_mem_handle *) malloc(sizeof(struct na_sm_mem_handle)); NA_CHECK_ERROR(na_sm_mem_handle == NULL, error, ret, NA_NOMEM, "Could not allocate NA SM memory handle"); @@ -3336,7 +3839,7 @@ na_sm_mem_handle_create(na_class_t NA_UNUSED *na_class, void *buf, na_sm_mem_handle->iov->iov_base = buf; na_sm_mem_handle->iov->iov_len = buf_size; na_sm_mem_handle->iovcnt = 1; - na_sm_mem_handle->flags = flags; + na_sm_mem_handle->flags = flags & 0xff; na_sm_mem_handle->len = buf_size; *mem_handle = (na_mem_handle_t) na_sm_mem_handle; @@ -3367,13 +3870,13 @@ na_sm_mem_handle_create_segments(na_class_t NA_UNUSED *na_class, NA_CHECK_ERROR(segment_count > iov_max, error, ret, NA_INVALID_ARG, "Segment count exceeds IOV_MAX limit"); - na_sm_mem_handle = (struct na_sm_mem_handle *) malloc( - sizeof(struct na_sm_mem_handle)); + na_sm_mem_handle = + (struct na_sm_mem_handle *) malloc(sizeof(struct na_sm_mem_handle)); NA_CHECK_ERROR(na_sm_mem_handle == NULL, error, ret, NA_NOMEM, "Could not allocate NA SM memory handle"); - na_sm_mem_handle->iov = (struct iovec *) malloc( - segment_count * sizeof(struct iovec)); + na_sm_mem_handle->iov = + (struct iovec *) malloc(segment_count * sizeof(struct iovec)); NA_CHECK_ERROR(na_sm_mem_handle->iov == NULL, error, ret, NA_NOMEM, "Could not allocate iovec"); @@ -3384,7 +3887,7 @@ na_sm_mem_handle_create_segments(na_class_t NA_UNUSED *na_class, na_sm_mem_handle->len += na_sm_mem_handle->iov[i].iov_len; } na_sm_mem_handle->iovcnt = segment_count; - na_sm_mem_handle->flags = flags; + na_sm_mem_handle->flags = flags & 0xff; *mem_handle = (na_mem_handle_t) na_sm_mem_handle; @@ -3401,8 +3904,8 @@ na_sm_mem_handle_create_segments(na_class_t NA_UNUSED *na_class, /*---------------------------------------------------------------------------*/ static na_return_t -na_sm_mem_handle_free(na_class_t NA_UNUSED *na_class, - na_mem_handle_t mem_handle) +na_sm_mem_handle_free( + na_class_t NA_UNUSED *na_class, na_mem_handle_t mem_handle) { struct na_sm_mem_handle *na_sm_mem_handle = (struct na_sm_mem_handle *) mem_handle; @@ -3416,8 +3919,8 @@ na_sm_mem_handle_free(na_class_t NA_UNUSED *na_class, /*---------------------------------------------------------------------------*/ static NA_INLINE na_size_t -na_sm_mem_handle_get_serialize_size(na_class_t NA_UNUSED *na_class, - na_mem_handle_t mem_handle) +na_sm_mem_handle_get_serialize_size( + na_class_t NA_UNUSED *na_class, na_mem_handle_t mem_handle) { struct na_sm_mem_handle *na_sm_mem_handle = (struct na_sm_mem_handle *) mem_handle; @@ -3436,7 +3939,7 @@ na_sm_mem_handle_serialize(na_class_t NA_UNUSED *na_class, void *buf, na_size_t NA_UNUSED buf_size, na_mem_handle_t mem_handle) { struct na_sm_mem_handle *na_sm_mem_handle = - (struct na_sm_mem_handle*) mem_handle; + (struct na_sm_mem_handle *) mem_handle; char *buf_ptr = (char *) buf; na_return_t ret = NA_SUCCESS; unsigned long i; @@ -3474,8 +3977,8 @@ na_sm_mem_handle_deserialize(na_class_t NA_UNUSED *na_class, na_return_t ret = NA_SUCCESS; unsigned long i; - na_sm_mem_handle = (struct na_sm_mem_handle *) malloc( - sizeof(struct na_sm_mem_handle)); + na_sm_mem_handle = + (struct na_sm_mem_handle *) malloc(sizeof(struct na_sm_mem_handle)); NA_CHECK_ERROR(na_sm_mem_handle == NULL, error, ret, NA_NOMEM, "Could not allocate NA SM memory handle"); na_sm_mem_handle->iov = NULL; @@ -3495,8 +3998,8 @@ na_sm_mem_handle_deserialize(na_class_t NA_UNUSED *na_class, buf_ptr += sizeof(size_t); /* Segments */ - na_sm_mem_handle->iov = (struct iovec *) malloc(na_sm_mem_handle->iovcnt * - sizeof(struct iovec)); + na_sm_mem_handle->iov = (struct iovec *) malloc( + na_sm_mem_handle->iovcnt * sizeof(struct iovec)); NA_CHECK_ERROR(na_sm_mem_handle->iov == NULL, error, ret, NA_NOMEM, "Could not allocate iovec"); @@ -3546,8 +4049,8 @@ na_sm_put(na_class_t *na_class, na_context_t *context, na_cb_t callback, #endif #if !defined(NA_SM_HAS_CMA) && !defined(__APPLE__) - NA_GOTO_ERROR(done, ret, NA_PROTOCOL_ERROR, - "Not implemented for this platform"); + NA_GOTO_ERROR( + done, ret, NA_OPNOTSUPPORTED, "Not implemented for this platform"); #endif switch (na_sm_mem_handle_remote->flags) { @@ -3559,8 +4062,8 @@ na_sm_put(na_class_t *na_class, na_context_t *context, na_cb_t callback, case NA_MEM_READWRITE: break; default: - NA_GOTO_ERROR(done, ret, NA_INVALID_ARG, - "Invalid memory access flag"); + NA_GOTO_ERROR( + done, ret, NA_INVALID_ARG, "Invalid memory access flag"); } /* Check op_id */ @@ -3569,8 +4072,9 @@ na_sm_put(na_class_t *na_class, na_context_t *context, na_cb_t callback, done, ret, NA_INVALID_ARG, "Invalid operation ID"); na_sm_op_id = (struct na_sm_op_id *) *op_id; - NA_CHECK_ERROR(!(hg_atomic_get32(&na_sm_op_id->status) & NA_SM_OP_COMPLETED), - done, ret, NA_BUSY, "Attempting to use OP ID that was not completed"); + NA_CHECK_ERROR( + !(hg_atomic_get32(&na_sm_op_id->status) & NA_SM_OP_COMPLETED), done, + ret, NA_BUSY, "Attempting to use OP ID that was not completed"); /* Make sure op ID is fully released before re-using it */ while (hg_atomic_cas32(&na_sm_op_id->ref_count, 1, 2) != HG_UTIL_TRUE) cpu_spinwait(); @@ -3586,8 +4090,8 @@ na_sm_put(na_class_t *na_class, na_context_t *context, na_cb_t callback, /* Translate local offset, skip this step if not necessary */ if (local_offset || length != na_sm_mem_handle_local->len) { local_iov = (struct iovec *) local_iovs; - na_sm_offset_translate(na_sm_mem_handle_local, local_offset, length, - local_iov, &liovcnt); + na_sm_offset_translate( + na_sm_mem_handle_local, local_offset, length, local_iov, &liovcnt); NA_LOG_DEBUG("Translated local offsets into %lu segment(s)", liovcnt); } else { local_iov = na_sm_mem_handle_local->iov; @@ -3607,22 +4111,22 @@ na_sm_put(na_class_t *na_class, na_context_t *context, na_cb_t callback, #if defined(NA_SM_HAS_CMA) nwrite = process_vm_writev(na_sm_addr->pid, local_iov, liovcnt, remote_iov, - riovcnt, /* unused */0); - NA_CHECK_ERROR(nwrite < 0, error, ret, NA_PROTOCOL_ERROR, + riovcnt, /* unused */ 0); + NA_CHECK_ERROR(nwrite < 0, error, ret, na_sm_errno_to_na(errno), "process_vm_writev() failed (%s)", strerror(errno)); - NA_CHECK_ERROR((na_size_t)nwrite != length, error, ret, NA_MSGSIZE, + NA_CHECK_ERROR((na_size_t) nwrite != length, error, ret, NA_MSGSIZE, "Wrote %ld bytes, was expecting %lu bytes", nwrite, length); #elif defined(__APPLE__) kret = task_for_pid(mach_task_self(), na_sm_addr->pid, &remote_task); - NA_CHECK_ERROR(kret != KERN_SUCCESS, error, ret, NA_PROTOCOL_ERROR, + NA_CHECK_ERROR(kret != KERN_SUCCESS, error, ret, NA_PERMISSION, "task_for_pid() failed (%s)\n" "Permission must be set to access remote memory, please refer to the " - "documentation for instructions.", mach_error_string(kret)); - NA_CHECK_ERROR(liovcnt > 1 || riovcnt > 1, error, ret, NA_PROTOCOL_ERROR, + "documentation for instructions.", + mach_error_string(kret)); + NA_CHECK_ERROR(liovcnt > 1 || riovcnt > 1, error, ret, NA_OPNOTSUPPORTED, "Non-contiguous transfers are not supported"); - kret = mach_vm_write(remote_task, - (mach_vm_address_t) remote_iov->iov_base, + kret = mach_vm_write(remote_task, (mach_vm_address_t) remote_iov->iov_base, (mach_vm_address_t) local_iov->iov_base, (mach_msg_type_number_t) length); NA_CHECK_ERROR(kret != KERN_SUCCESS, error, ret, NA_PROTOCOL_ERROR, @@ -3630,16 +4134,10 @@ na_sm_put(na_class_t *na_class, na_context_t *context, na_cb_t callback, #endif /* Immediate completion */ - ret = na_sm_complete(na_sm_op_id); + ret = na_sm_complete( + na_sm_op_id, NA_SM_CLASS(na_class)->endpoint.source_addr->tx_notify); NA_CHECK_NA_ERROR(error, ret, "Could not complete operation"); - /* Notify local completion */ - if (!NA_SM_CLASS(na_class)->no_wait) { - int rc = hg_event_set(NA_SM_CLASS(na_class)->self_addr->local_notify); - NA_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, NA_PROTOCOL_ERROR, - "Could not signal local completion"); - } - done: return ret; @@ -3678,8 +4176,8 @@ na_sm_get(na_class_t *na_class, na_context_t *context, na_cb_t callback, #endif #if !defined(NA_SM_HAS_CMA) && !defined(__APPLE__) - NA_GOTO_ERROR(done, ret, NA_PROTOCOL_ERROR, - "Not implemented for this platform"); + NA_GOTO_ERROR( + done, ret, NA_OPNOTSUPPORTED, "Not implemented for this platform"); #endif switch (na_sm_mem_handle_remote->flags) { @@ -3691,13 +4189,14 @@ na_sm_get(na_class_t *na_class, na_context_t *context, na_cb_t callback, case NA_MEM_READWRITE: break; default: - NA_GOTO_ERROR(done, ret, NA_INVALID_ARG, - "Invalid memory access flag"); + NA_GOTO_ERROR( + done, ret, NA_INVALID_ARG, "Invalid memory access flag"); } na_sm_op_id = (struct na_sm_op_id *) *op_id; - NA_CHECK_ERROR(!(hg_atomic_get32(&na_sm_op_id->status) & NA_SM_OP_COMPLETED), - done, ret, NA_BUSY, "Attempting to use OP ID that was not completed"); + NA_CHECK_ERROR( + !(hg_atomic_get32(&na_sm_op_id->status) & NA_SM_OP_COMPLETED), done, + ret, NA_BUSY, "Attempting to use OP ID that was not completed"); /* Make sure op ID is fully released before re-using it */ while (hg_atomic_cas32(&na_sm_op_id->ref_count, 1, 2) != HG_UTIL_TRUE) cpu_spinwait(); @@ -3713,8 +4212,8 @@ na_sm_get(na_class_t *na_class, na_context_t *context, na_cb_t callback, /* Translate local offset, skip this step if not necessary */ if (local_offset || length != na_sm_mem_handle_local->len) { local_iov = (struct iovec *) local_iovs; - na_sm_offset_translate(na_sm_mem_handle_local, local_offset, length, - local_iov, &liovcnt); + na_sm_offset_translate( + na_sm_mem_handle_local, local_offset, length, local_iov, &liovcnt); NA_LOG_DEBUG("Translated local offsets into %lu segment(s)", liovcnt); } else { local_iov = na_sm_mem_handle_local->iov; @@ -3734,16 +4233,17 @@ na_sm_get(na_class_t *na_class, na_context_t *context, na_cb_t callback, #if defined(NA_SM_HAS_CMA) nread = process_vm_readv(na_sm_addr->pid, local_iov, liovcnt, remote_iov, - riovcnt, /* unused */0); - NA_CHECK_ERROR(nread < 0, error, ret, NA_PROTOCOL_ERROR, + riovcnt, /* unused */ 0); + NA_CHECK_ERROR(nread < 0, error, ret, na_sm_errno_to_na(errno), "process_vm_readv() failed (%s)", strerror(errno)); #elif defined(__APPLE__) kret = task_for_pid(mach_task_self(), na_sm_addr->pid, &remote_task); - NA_CHECK_ERROR(kret != KERN_SUCCESS, error, ret, NA_PROTOCOL_ERROR, + NA_CHECK_ERROR(kret != KERN_SUCCESS, error, ret, NA_PERMISSION, "task_for_pid() failed (%s)\n" "Permission must be set to access remote memory, please refer to the " - "documentation for instructions.", mach_error_string(kret)); - NA_CHECK_ERROR(liovcnt > 1 || riovcnt > 1, error, ret, NA_PROTOCOL_ERROR, + "documentation for instructions.", + mach_error_string(kret)); + NA_CHECK_ERROR(liovcnt > 1 || riovcnt > 1, error, ret, NA_OPNOTSUPPORTED, "Non-contiguous transfers are not supported"); kret = mach_vm_read_overwrite(remote_task, @@ -3753,21 +4253,15 @@ na_sm_get(na_class_t *na_class, na_context_t *context, na_cb_t callback, "mach_vm_read_overwrite() failed (%s)", mach_error_string(kret)); #endif #if defined(NA_SM_HAS_CMA) || defined(__APPLE__) - NA_CHECK_ERROR((na_size_t)nread != length, error, ret, NA_MSGSIZE, + NA_CHECK_ERROR((na_size_t) nread != length, error, ret, NA_MSGSIZE, "Read %ld bytes, was expecting %lu bytes", nread, length); #endif /* Immediate completion */ - ret = na_sm_complete(na_sm_op_id); + ret = na_sm_complete( + na_sm_op_id, NA_SM_CLASS(na_class)->endpoint.source_addr->tx_notify); NA_CHECK_NA_ERROR(error, ret, "Could not complete operation"); - /* Notify local completion */ - if (!NA_SM_CLASS(na_class)->no_wait) { - int rc = hg_event_set(NA_SM_CLASS(na_class)->self_addr->local_notify); - NA_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, NA_PROTOCOL_ERROR, - "Could not signal local completion"); - } - done: return ret; @@ -3784,10 +4278,13 @@ na_sm_poll_get_fd(na_class_t *na_class, na_context_t NA_UNUSED *context) { int fd = -1; - fd = hg_poll_get_fd(NA_SM_CLASS(na_class)->poll_set); - NA_CHECK_ERROR_NORET(fd == -1, done, "Could not get poll fd from poll set"); + if (NA_SM_CLASS(na_class)->endpoint.poll_set) { + fd = hg_poll_get_fd(NA_SM_CLASS(na_class)->endpoint.poll_set); + NA_CHECK_ERROR_NORET( + fd == -1, done, "Could not get poll fd from poll set"); + } - done: +done: return fd; } @@ -3795,57 +4292,114 @@ na_sm_poll_get_fd(na_class_t *na_class, na_context_t NA_UNUSED *context) static NA_INLINE na_bool_t na_sm_poll_try_wait(na_class_t *na_class, na_context_t NA_UNUSED *context) { + struct na_sm_endpoint *na_sm_endpoint = &NA_SM_CLASS(na_class)->endpoint; + struct na_sm_addr_list *poll_addr_list = &na_sm_endpoint->poll_addr_list; struct na_sm_addr *na_sm_addr; - na_bool_t ret = NA_TRUE; - - /* Check whether something is in one of the ring buffers */ - hg_thread_spin_lock(&NA_SM_CLASS(na_class)->poll_addr_queue_lock); - HG_QUEUE_FOREACH(na_sm_addr, &NA_SM_CLASS(na_class)->poll_addr_queue, - poll_entry) { - if (!na_sm_ring_buf_is_empty(na_sm_addr->na_sm_recv_ring_buf)) { - ret = NA_FALSE; - break; + + /* Check whether something is in one of the rx queues */ + hg_thread_spin_lock(&poll_addr_list->lock); + HG_LIST_FOREACH (na_sm_addr, &poll_addr_list->list, entry) { + if (!na_sm_msg_queue_is_empty(na_sm_addr->rx_queue)) { + hg_thread_spin_unlock(&poll_addr_list->lock); + return NA_FALSE; } } - hg_thread_spin_unlock(&NA_SM_CLASS(na_class)->poll_addr_queue_lock); + hg_thread_spin_unlock(&poll_addr_list->lock); - return ret; + /* Check whether something is in the cmd queue */ + if (na_sm_endpoint->source_addr->shared_region && + !na_sm_cmd_queue_is_empty( + &na_sm_endpoint->source_addr->shared_region->cmd_queue)) { + return NA_FALSE; + } + + return NA_TRUE; } /*---------------------------------------------------------------------------*/ static na_return_t -na_sm_progress(na_class_t *na_class, na_context_t NA_UNUSED *context, - unsigned int timeout) +na_sm_progress( + na_class_t *na_class, na_context_t NA_UNUSED *context, unsigned int timeout) { - double remaining = timeout / 1000.0; /* Convert timeout in ms into seconds */ + struct na_sm_endpoint *na_sm_endpoint = &NA_SM_CLASS(na_class)->endpoint; + double remaining = + timeout / 1000.0; /* Convert timeout in ms into seconds */ na_return_t ret = NA_TIMEOUT; do { + struct hg_poll_event events[NA_SM_MAX_EVENTS] = {0}; + unsigned int nevents = 0, i; + na_bool_t progressed = NA_FALSE; hg_time_t t1, t2; - hg_util_bool_t progressed; - unsigned int poll_timeout = (NA_SM_CLASS(na_class)->no_wait) ? 0 : - (unsigned int) (remaining * 1000.0); - int rc; if (timeout) hg_time_get_current(&t1); - rc = hg_poll_wait(NA_SM_CLASS(na_class)->poll_set, poll_timeout, - &progressed); - NA_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, NA_PROTOCOL_ERROR, - "hg_poll_wait() failed"); + if (na_sm_endpoint->poll_set) { + int rc = hg_poll_wait(na_sm_endpoint->poll_set, + (unsigned int) (remaining * 1000.0), NA_SM_MAX_EVENTS, events, + &nevents); + NA_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, + na_sm_errno_to_na(errno), "hg_poll_wait() failed"); + + /* Process events */ + for (i = 0; i < nevents; i++) { + if (events[i].progressed && events[i].ptr) { + na_bool_t progressed_rx; + + ret = na_sm_progress_rx_queue( + na_sm_endpoint, events[i].ptr, &progressed_rx); + NA_CHECK_NA_ERROR(done, ret, "Could not progress rx queue"); + progressed |= progressed_rx; + } + progressed |= events[i].progressed; + } + } else { + struct na_sm_addr_list *poll_addr_list = + &na_sm_endpoint->poll_addr_list; + struct na_sm_addr *na_sm_addr; + + /* Check whether something is in one of the rx queues */ + hg_thread_spin_lock(&poll_addr_list->lock); + HG_LIST_FOREACH (na_sm_addr, &poll_addr_list->list, entry) { + na_bool_t progressed_rx; - /* We progressed, return success */ - if (progressed) { - ret = NA_SUCCESS; - break; + ret = na_sm_progress_rx_queue( + na_sm_endpoint, na_sm_addr, &progressed_rx); + NA_CHECK_NA_ERROR(done, ret, "Could not progress rx queue"); + + progressed |= progressed_rx; + } + hg_thread_spin_unlock(&poll_addr_list->lock); + + /* Look for message in cmd queue (if listening) */ + if (na_sm_endpoint->source_addr->shared_region) { + na_sm_cmd_hdr_t cmd_hdr = {.val = 0}; + + while (na_sm_cmd_queue_pop( + &na_sm_endpoint->source_addr->shared_region->cmd_queue, + &cmd_hdr)) { + ret = na_sm_process_cmd(na_sm_endpoint, + NA_SM_CLASS(na_class)->username, cmd_hdr, -1, -1); + NA_CHECK_NA_ERROR(done, ret, "Could not process cmd"); + progressed |= NA_TRUE; + } + } } + /* Process retries */ + ret = na_sm_process_retries(&na_sm_endpoint->retry_op_queue); + NA_CHECK_NA_ERROR(done, ret, "Could not process retried msgs"); + if (timeout) { hg_time_get_current(&t2); remaining -= hg_time_to_double(hg_time_subtract(t2, t1)); } - } while ((int)(remaining * 1000.0) > 0); + + if (nevents == 0 || !progressed) + ret = NA_TIMEOUT; /* Return NA_TIMEOUT if no events */ + + } while (remaining > 0 && (ret != NA_SUCCESS)); done: return ret; @@ -3853,60 +4407,33 @@ na_sm_progress(na_class_t *na_class, na_context_t NA_UNUSED *context, /*---------------------------------------------------------------------------*/ static na_return_t -na_sm_cancel(na_class_t *na_class, na_context_t NA_UNUSED *context, - na_op_id_t op_id) +na_sm_cancel( + na_class_t *na_class, na_context_t NA_UNUSED *context, na_op_id_t op_id) { + struct na_sm_op_queue *op_queue = NULL; struct na_sm_op_id *na_sm_op_id = (struct na_sm_op_id *) op_id; na_return_t ret = NA_SUCCESS; - na_bool_t canceled = NA_FALSE; /* Exit if op has already completed */ - if (hg_atomic_or32(&na_sm_op_id->status, NA_SM_OP_CANCELED) - & NA_SM_OP_COMPLETED) + if (hg_atomic_or32(&na_sm_op_id->status, NA_SM_OP_CANCELED) & + NA_SM_OP_COMPLETED) goto done; NA_LOG_DEBUG("Canceling operation ID %p", na_sm_op_id); switch (na_sm_op_id->completion_data.callback_info.type) { - case NA_CB_LOOKUP: - /* Nothing */ - break; case NA_CB_RECV_UNEXPECTED: - /* Must remove op_id from unexpected op_id queue */ - hg_thread_spin_lock( - &NA_SM_CLASS(na_class)->unexpected_op_queue_lock); - if (hg_atomic_get32(&na_sm_op_id->status) & NA_SM_OP_QUEUED) { - HG_QUEUE_REMOVE(&NA_SM_CLASS(na_class)->unexpected_op_queue, - na_sm_op_id, na_sm_op_id, entry); - hg_atomic_and32(&na_sm_op_id->status, ~NA_SM_OP_QUEUED); - canceled = NA_TRUE; - } - hg_thread_spin_unlock( - &NA_SM_CLASS(na_class)->unexpected_op_queue_lock); + /* Must remove op_id from unexpected op queue */ + op_queue = &NA_SM_CLASS(na_class)->endpoint.unexpected_op_queue; break; case NA_CB_RECV_EXPECTED: - /* Must remove op_id from unexpected op_id queue */ - hg_thread_spin_lock(&NA_SM_CLASS(na_class)->expected_op_queue_lock); - if (hg_atomic_get32(&na_sm_op_id->status) & NA_SM_OP_QUEUED) { - HG_QUEUE_REMOVE(&NA_SM_CLASS(na_class)->expected_op_queue, - na_sm_op_id, na_sm_op_id, entry); - hg_atomic_and32(&na_sm_op_id->status, ~NA_SM_OP_QUEUED); - canceled = NA_TRUE; - } - hg_thread_spin_unlock( - &NA_SM_CLASS(na_class)->expected_op_queue_lock); + /* Must remove op_id from unexpected op queue */ + op_queue = &NA_SM_CLASS(na_class)->endpoint.expected_op_queue; break; case NA_CB_SEND_UNEXPECTED: case NA_CB_SEND_EXPECTED: - /* Must remove op_id from retry op_id queue */ - hg_thread_spin_lock(&NA_SM_CLASS(na_class)->retry_op_queue_lock); - if (hg_atomic_get32(&na_sm_op_id->status) & NA_SM_OP_QUEUED) { - HG_QUEUE_REMOVE(&NA_SM_CLASS(na_class)->retry_op_queue, - na_sm_op_id, na_sm_op_id, entry); - hg_atomic_and32(&na_sm_op_id->status, ~NA_SM_OP_QUEUED); - canceled = NA_TRUE; - } - hg_thread_spin_unlock(&NA_SM_CLASS(na_class)->retry_op_queue_lock); + /* Must remove op_id from retry op queue */ + op_queue = &NA_SM_CLASS(na_class)->endpoint.retry_op_queue; break; case NA_CB_PUT: /* Nothing */ @@ -3920,10 +4447,24 @@ na_sm_cancel(na_class_t *na_class, na_context_t NA_UNUSED *context, na_sm_op_id->completion_data.callback_info.type); } - /* Cancel op id */ - if (canceled) { - ret = na_sm_complete(na_sm_op_id); - NA_CHECK_NA_ERROR(done, ret, "Could not complete operation"); + /* Remove op id from queue it is on */ + if (op_queue) { + na_bool_t canceled = NA_FALSE; + + hg_thread_spin_lock(&op_queue->lock); + if (hg_atomic_get32(&na_sm_op_id->status) & NA_SM_OP_QUEUED) { + HG_QUEUE_REMOVE(&op_queue->queue, na_sm_op_id, na_sm_op_id, entry); + hg_atomic_and32(&na_sm_op_id->status, ~NA_SM_OP_QUEUED); + canceled = NA_TRUE; + } + hg_thread_spin_unlock(&op_queue->lock); + + /* Cancel op id */ + if (canceled) { + ret = na_sm_complete(na_sm_op_id, + NA_SM_CLASS(na_class)->endpoint.source_addr->tx_notify); + NA_CHECK_NA_ERROR(done, ret, "Could not complete operation"); + } } done: diff --git a/src/na/na_types.h b/src/na/na_types.h index 9245e0fd..483e1c35 100644 --- a/src/na/na_types.h +++ b/src/na/na_types.h @@ -55,6 +55,7 @@ struct na_segment { X(NA_ACCESS) /*!< permission denied */ \ X(NA_FAULT) /*!< bad address */ \ X(NA_BUSY) /*!< device or resource busy */ \ + X(NA_EXIST) /*!< entry already exists */ \ X(NA_NODEV) /*!< no such device */ \ X(NA_INVALID_ARG) /*!< invalid argument */ \ X(NA_PROTOCOL_ERROR) /*!< protocol error */ \ From 6762ff57c8ea9ed66d2ac8321aa6dec4b2f23b07 Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Mon, 1 Jun 2020 23:48:29 -0500 Subject: [PATCH 12/30] Testing: add concurrent lookup test Clean up test server and benchmarks --- Testing/CMakeLists.txt | 1 + Testing/na/CMakeLists.txt | 6 +- Testing/test_lookup.c | 172 ++++++++++++++++++++++++++++++++++++++ Testing/test_read_bw.c | 3 +- Testing/test_server.c | 8 +- Testing/test_write_bw.c | 3 +- 6 files changed, 184 insertions(+), 9 deletions(-) create mode 100644 Testing/test_lookup.c diff --git a/Testing/CMakeLists.txt b/Testing/CMakeLists.txt index b4014278..a6d3338a 100644 --- a/Testing/CMakeLists.txt +++ b/Testing/CMakeLists.txt @@ -305,6 +305,7 @@ set(MERCURY_TESTS rpc bulk ) +build_mercury_test(lookup) # List of serial tests set(MERCURY_SERIAL_TESTS diff --git a/Testing/na/CMakeLists.txt b/Testing/na/CMakeLists.txt index 96587b4f..04466fce 100644 --- a/Testing/na/CMakeLists.txt +++ b/Testing/na/CMakeLists.txt @@ -71,11 +71,11 @@ endif() #------------------------------------------------------------------------------ # Network abstraction test -build_na_test(client) +#build_na_test(client) build_na_test(server) -build_na_test(cancel_client) +#build_na_test(cancel_client) build_na_test(cancel_server) -build_na_test(lat_client) +#build_na_test(lat_client) build_na_test(lat_server) #------------------------------------------------------------------------------ diff --git a/Testing/test_lookup.c b/Testing/test_lookup.c new file mode 100644 index 00000000..ac690f06 --- /dev/null +++ b/Testing/test_lookup.c @@ -0,0 +1,172 @@ +/* + * Copyright (C) 2013-2019 Argonne National Laboratory, Department of Energy, + * UChicago Argonne, LLC and The HDF Group. + * All rights reserved. + * + * The full copyright notice, including terms governing use, modification, + * and redistribution, is contained in the COPYING file that can be + * found at the root of the source code distribution tree. + */ + +#include "mercury_test.h" + +#include +#include + +/****************/ +/* Local Macros */ +/****************/ + +/************************************/ +/* Local Type and Struct Definition */ +/************************************/ + +#ifdef HG_TEST_HAS_THREAD_POOL +struct hg_test_thread_args { + struct hg_test_info *hg_test_info; + hg_thread_mutex_t test_mutex; + hg_thread_cond_t test_cond; + unsigned int n_threads; +}; +#endif + +/********************/ +/* Local Prototypes */ +/********************/ + +static hg_return_t +hg_test_rpc_lookup(hg_class_t *hg_class, const char *target_name); + +/*******************/ +/* Local Variables */ +/*******************/ + +/*---------------------------------------------------------------------------*/ +#ifdef HG_TEST_HAS_THREAD_POOL +static HG_THREAD_RETURN_TYPE +hg_test_lookup_thread(void *arg) +{ + struct hg_test_thread_args *hg_test_thread_args = + (struct hg_test_thread_args *) arg; + struct hg_test_info *hg_test_info = hg_test_thread_args->hg_test_info; + hg_return_t hg_ret; + + /* Wait for all threads to have reached that point */ + hg_thread_mutex_lock(&hg_test_thread_args->test_mutex); + if (++hg_test_thread_args->n_threads == HG_TEST_NUM_THREADS_DEFAULT) + hg_thread_cond_broadcast(&hg_test_thread_args->test_cond); + hg_thread_mutex_unlock(&hg_test_thread_args->test_mutex); + + hg_thread_mutex_lock(&hg_test_thread_args->test_mutex); + while (hg_test_thread_args->n_threads != HG_TEST_NUM_THREADS_DEFAULT) + hg_thread_cond_wait(&hg_test_thread_args->test_cond, + &hg_test_thread_args->test_mutex); + hg_thread_mutex_unlock(&hg_test_thread_args->test_mutex); + + HG_TEST_LOG_DEBUG("Now doing lookup in loop"); + hg_ret = hg_test_rpc_lookup(hg_test_info->hg_class, + hg_test_info->na_test_info.target_name); + HG_TEST_CHECK_ERROR_NORET(hg_ret != HG_SUCCESS, done, "lookup test failed"); + +done: + return NULL; +} +#endif + +/*---------------------------------------------------------------------------*/ +static hg_return_t +hg_test_rpc_lookup(hg_class_t *hg_class, const char *target_name) +{ + hg_return_t ret = HG_SUCCESS; + hg_addr_t target_addr = HG_ADDR_NULL; + int i; + + for (i = 0; i < 32; i++) { + /* Forward call to remote addr and get a new request */ + ret = HG_Addr_lookup2(hg_class, target_name, &target_addr); + HG_TEST_CHECK_HG_ERROR(done, ret, "HG_Addr_lookup() failed (%s)", + HG_Error_to_string(ret)); + + ret = HG_Addr_set_remove(hg_class, target_addr); + HG_TEST_CHECK_HG_ERROR(error, ret, "HG_Addr_set_remove() failed (%s)", + HG_Error_to_string(ret)); + + ret = HG_Addr_free(hg_class, target_addr); + HG_TEST_CHECK_HG_ERROR(error, ret, "HG_Addr_free() failed (%s)", + HG_Error_to_string(ret)); + target_addr = HG_ADDR_NULL; + } + +done: + return ret; + +error: + HG_Addr_free(hg_class, target_addr); + + return ret; +} + +/*---------------------------------------------------------------------------*/ +int +main(int argc, char *argv[]) +{ + struct hg_test_info hg_test_info = { 0 }; +#ifdef HG_TEST_HAS_THREAD_POOL + struct hg_test_thread_args hg_test_thread_args; + hg_thread_t threads[HG_TEST_NUM_THREADS_DEFAULT]; + int i; +#endif + hg_return_t hg_ret; + int ret = EXIT_SUCCESS; + + /* Initialize the interface */ + hg_ret = HG_Test_init(argc, argv, &hg_test_info); + HG_TEST_CHECK_ERROR(hg_ret != HG_SUCCESS, done, ret, EXIT_FAILURE, + "HG_Test_init() failed"); + + HG_Addr_free(hg_test_info.hg_class, hg_test_info.target_addr); + hg_test_info.target_addr = HG_ADDR_NULL; + +#ifdef HG_TEST_HAS_THREAD_POOL + hg_test_thread_args.hg_test_info = &hg_test_info; + hg_thread_mutex_init(&hg_test_thread_args.test_mutex); + hg_thread_cond_init(&hg_test_thread_args.test_cond); + hg_test_thread_args.n_threads = 0; +#endif + + /* Create threads */ + HG_TEST("lookup RPC"); +#ifdef HG_TEST_HAS_THREAD_POOL + for (i = 0; i < HG_TEST_NUM_THREADS_DEFAULT; i++) + hg_thread_create(&threads[i], hg_test_lookup_thread, + &hg_test_thread_args); + + for (i = 0; i < HG_TEST_NUM_THREADS_DEFAULT; i++) + hg_thread_join(threads[i]); +#else + hg_ret = hg_test_rpc_lookup(hg_test_info.hg_class, + hg_test_info.na_test_info.target_name); + HG_TEST_CHECK_ERROR(hg_ret != HG_SUCCESS, done, ret, EXIT_FAILURE, + "lookup test failed"); +#endif + HG_PASSED(); + + hg_ret = HG_Addr_lookup2(hg_test_info.hg_class, + hg_test_info.na_test_info.target_name, &hg_test_info.target_addr); + HG_TEST_CHECK_ERROR(hg_ret != HG_SUCCESS, done, ret, EXIT_FAILURE, + "HG_Addr_lookup() failed (%s)", HG_Error_to_string(hg_ret)); + +done: + if (ret != EXIT_SUCCESS) + HG_FAILED(); + +#ifdef HG_TEST_HAS_THREAD_POOL + hg_thread_mutex_destroy(&hg_test_thread_args.test_mutex); + hg_thread_cond_destroy(&hg_test_thread_args.test_cond); +#endif + + hg_ret = HG_Test_finalize(&hg_test_info); + HG_TEST_CHECK_ERROR_DONE(hg_ret != HG_SUCCESS, "HG_Test_finalize() failed"); + + return ret; +} diff --git a/Testing/test_read_bw.c b/Testing/test_read_bw.c index b2c77278..81899e6a 100644 --- a/Testing/test_read_bw.c +++ b/Testing/test_read_bw.c @@ -265,7 +265,8 @@ main(int argc, char *argv[]) "measure_bulk_transfer() failed"); } - fprintf(stdout, "\n"); + if (hg_test_info.na_test_info.mpi_comm_rank == 0) + fprintf(stdout, "\n"); } done: diff --git a/Testing/test_server.c b/Testing/test_server.c index 752a9749..d0fe6f8b 100644 --- a/Testing/test_server.c +++ b/Testing/test_server.c @@ -37,9 +37,9 @@ struct hg_test_worker { /********************/ #ifdef HG_TEST_HAS_THREAD_POOL -static HG_INLINE HG_THREAD_RETURN_TYPE +static HG_THREAD_RETURN_TYPE hg_test_progress_thread(void *arg); -static HG_INLINE HG_THREAD_RETURN_TYPE +static HG_THREAD_RETURN_TYPE hg_test_progress_work(void *arg); #endif @@ -50,7 +50,7 @@ hg_test_progress_work(void *arg); /*---------------------------------------------------------------------------*/ #ifdef HG_TEST_HAS_THREAD_POOL -static HG_INLINE HG_THREAD_RETURN_TYPE +static HG_THREAD_RETURN_TYPE hg_test_progress_thread(void *arg) { hg_context_t *context = (hg_context_t *) arg; @@ -76,7 +76,7 @@ hg_test_progress_thread(void *arg) } /*---------------------------------------------------------------------------*/ -static HG_INLINE HG_THREAD_RETURN_TYPE +static HG_THREAD_RETURN_TYPE hg_test_progress_work(void *arg) { struct hg_test_worker *worker = (struct hg_test_worker *) arg; diff --git a/Testing/test_write_bw.c b/Testing/test_write_bw.c index 3b0ea6c8..a64fc6dd 100644 --- a/Testing/test_write_bw.c +++ b/Testing/test_write_bw.c @@ -265,7 +265,8 @@ main(int argc, char *argv[]) "measure_bulk_transfer() failed"); } - fprintf(stdout, "\n"); + if (hg_test_info.na_test_info.mpi_comm_rank == 0) + fprintf(stdout, "\n"); } done: From b3af5d2224a962d032d25521c8dfd722cf2059ee Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Thu, 4 Jun 2020 16:25:25 -0500 Subject: [PATCH 13/30] Update travis build scripts for new Asan/Tsan builds Update mercury_common ctest script Remove old jenkins scripts --- .travis.yml | 6 +- Testing/script/jenkins_build.sh | 47 --- Testing/script/jenkins_script.cmake | 128 -------- Testing/script/mercury_common.cmake | 474 ++++++++++++++++++---------- Testing/script/travis_script.cmake | 86 ++--- 5 files changed, 360 insertions(+), 381 deletions(-) delete mode 100755 Testing/script/jenkins_build.sh delete mode 100644 Testing/script/jenkins_script.cmake diff --git a/.travis.yml b/.travis.yml index aa36ddad..49dc2974 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,8 +14,6 @@ cache: - $HOME/install env: - global: - - MERCURY_NO_LOOP="true" jobs: - MERCURY_BUILD_CONFIGURATION="Debug" - MERCURY_BUILD_CONFIGURATION="RelWithDebInfo" @@ -35,7 +33,7 @@ jobs: - libtsan0 - libasan5 - liblsan0 - env: MERCURY_MEMORYCHECK_TYPE="ThreadSanitizer" + env: MERCURY_BUILD_CONFIGURATION="Tsan" - os: linux compiler: gcc addons: @@ -48,7 +46,7 @@ jobs: - libtsan0 - libasan5 - liblsan0 - env: MERCURY_MEMORYCHECK_TYPE="AddressSanitizer" + env: MERCURY_BUILD_CONFIGURATION="Asan" - os: linux compiler: gcc addons: diff --git a/Testing/script/jenkins_build.sh b/Testing/script/jenkins_build.sh deleted file mode 100755 index ebf800b0..00000000 --- a/Testing/script/jenkins_build.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/sh - -echo "Running build script from repository" -echo "(current dir is repo root: $PWD)" -set -# store the current directory in a local variable to get back to it later -MERCURY_WORKSPACE_DIR=$PWD - -cd $WORKSPACE - -# build bmi -git clone git://git.mcs.anl.gov/bmi bmi -pushd bmi -./prepare && ./configure --enable-shared --enable-bmi-only --prefix=$WORKSPACE/.install -make && make install -popd - -# build cci -CCI_VERSION=2.0 -mkdir cci -pushd cci -wget http://cci-forum.com/wp-content/uploads/2016/06/cci-${CCI_VERSION}.tar.gz -tar -xzf cci-${CCI_VERSION}.tar.gz -pushd cci-${CCI_VERSION} -patch -p1 < $MERCURY_WORKSPACE_DIR/Testing/script/cci_sm.patch -./configure --prefix=$WORKSPACE/.install --enable-static && make && make install -popd -popd - -# echo mpi commands needed to compile -echo "mpicc -show" -mpicc -show - -# set up testing configuration -export MERCURY_BUILD_CONFIGURATION="Debug" -export MERCURY_DASHBOARD_MODEL="Nightly" -export MERCURY_DO_COVERAGE="true" -export MERCURY_DO_MEMCHECK="false" - -# export variable needed for bmi testing -export MERCURY_PORT_NAME='tcp://localhost:3344' - -# get back to the testing script location -pushd $MERCURY_WORKSPACE_DIR/Testing/script -ctest -S jenkins_script.cmake -VV 2>&1 -popd - diff --git a/Testing/script/jenkins_script.cmake b/Testing/script/jenkins_script.cmake deleted file mode 100644 index 3d9c51da..00000000 --- a/Testing/script/jenkins_script.cmake +++ /dev/null @@ -1,128 +0,0 @@ -# This script takes in optional environment variables. -# MERCURY_BUILD_CONFIGURATION=Debug | Release -# MERCURY_DASHBOARD_MODEL=Experimental | Nightly | Continuous -# MERCURY_BUILD_STATIC_LIBRARIES -# MERCURY_DO_COVERAGE -# MERCURY_DO_MEMCHECK - -# MERCURY_BUILD_CONFIGURATION = Debug | Release -set(MERCURY_BUILD_CONFIGURATION "$ENV{MERCURY_BUILD_CONFIGURATION}") -if(NOT MERCURY_BUILD_CONFIGURATION) - set(MERCURY_BUILD_CONFIGURATION "Debug") -endif() -string(TOLOWER ${MERCURY_BUILD_CONFIGURATION} lower_mercury_build_configuration) -set(CTEST_BUILD_CONFIGURATION ${MERCURY_BUILD_CONFIGURATION}) - -# MERCURY_DASHBOARD_MODEL=Experimental | Nightly | Continuous -set(MERCURY_DASHBOARD_MODEL "$ENV{MERCURY_DASHBOARD_MODEL}") -if(NOT MERCURY_DASHBOARD_MODEL) - set(MERCURY_DASHBOARD_MODEL "Experimental") -endif() -set(dashboard_model ${MERCURY_DASHBOARD_MODEL}) -string(TOLOWER ${MERCURY_DASHBOARD_MODEL} lower_mercury_dashboard_model) - -# Disable loop when MERCURY_DASHBOARD_MODEL=Continuous -set(MERCURY_NO_LOOP $ENV{MERCURY_NO_LOOP}) -if(MERCURY_NO_LOOP) - message("Disabling looping (if applicable)") - set(dashboard_disable_loop TRUE) -endif() - -# Number of jobs to build -set(CTEST_BUILD_FLAGS "-j4") - -# Build shared libraries -set(mercury_build_shared ON) -set(MERCURY_BUILD_STATIC_LIBRARIES $ENV{MERCURY_BUILD_STATIC_LIBRARIES}) -if(MERCURY_BUILD_STATIC_LIBRARIES) - message("Building static libraries") - set(mercury_build_shared OFF) -endif() - -# Build name referenced in cdash -set(CTEST_BUILD_NAME "jenkins-x64-${lower_mercury_dashboard_model}") - -set(CTEST_CMAKE_GENERATOR "Unix Makefiles") -# Must point to the root where we can checkout/build/run the tests -set(CTEST_DASHBOARD_ROOT "$ENV{WORKSPACE}/${MERCURY_DASHBOARD_MODEL}") -# Give a site name -set(CTEST_SITE "$ENV{NODE_NAME}") -set(CTEST_TEST_TIMEOUT 180) # 3 minute timeout - -# Optional coverage options -set(MERCURY_DO_COVERAGE $ENV{MERCURY_DO_COVERAGE}) -if(MERCURY_DO_COVERAGE) - message("Enabling Coverage") - set(CTEST_COVERAGE_COMMAND "/usr/bin/gcov") - set(CTEST_BUILD_NAME "${CTEST_BUILD_NAME}-coverage") - # don't run parallel coverage tests, no matter what. - set(CTEST_TEST_ARGS PARALLEL_LEVEL 1) - - # needed by mercury_common.cmake - set(dashboard_do_coverage TRUE) - - # add Coverage dir to the root so that we don't mess the non-coverage - # dashboard. - set(CTEST_DASHBOARD_ROOT "${CTEST_DASHBOARD_ROOT}/Coverage") -endif() - -# Optional memcheck options -set(MERCURY_DO_MEMCHECK $ENV{MERCURY_DO_MEMCHECK}) -if(MERCURY_DO_MEMCHECK) - message("Enabling Memcheck") - set(CTEST_MEMORYCHECK_COMMAND "/home/soumagne/apps/valgrind/default/bin/valgrind") - set(CTEST_MEMORYCHECK_COMMAND_OPTIONS "--gen-suppressions=all --trace-children=yes --fair-sched=yes -q --leak-check=yes --show-reachable=yes --num-callers=50 -v") - #set(CTEST_MEMORYCHECK_SUPPRESSIONS_FILE ${CTEST_SCRIPT_DIRECTORY}/MercuryValgrindSuppressions.supp) - - # needed by mercury_common.cmake - set(dashboard_do_memcheck TRUE) -endif() - -set(dashboard_source_name mercury) -set(dashboard_binary_name mercury-${lower_mercury_build_configuration}) -if(NOT mercury_build_shared) - set(dashboard_binary_name ${dashboard_binary_name}-static) -endif() - -# Initial cache used to build mercury, options can be modified here -set(dashboard_cache " -CMAKE_C_FLAGS:STRING=-Wall -Wextra -Wshadow -Winline -Wundef -Wcast-qual -std=gnu99 - -BUILD_SHARED_LIBS:BOOL=${mercury_build_shared} -BUILD_TESTING:BOOL=ON - -MEMORYCHECK_COMMAND:FILEPATH=${CTEST_MEMORYCHECK_COMMAND} -MEMORYCHECK_SUPPRESSIONS_FILE:FILEPATH=${CTEST_MEMORYCHECK_SUPPRESSIONS_FILE} -COVERAGE_COMMAND:FILEPATH=${CTEST_COVERAGE_COMMAND} - -MERCURY_ENABLE_COVERAGE:BOOL=${dashboard_do_coverage} -MERCURY_ENABLE_PARALLEL_TESTING:BOOL=ON -MERCURY_USE_BOOST_PP:BOOL=OFF -MERCURY_USE_XDR:BOOL=OFF -NA_USE_BMI:BOOL=ON -BMI_INCLUDE_DIR:PATH=$ENV{WORKSPACE}/.install/include -BMI_LIBRARY:FILEPATH=$ENV{WORKSPACE}/.install/lib/libbmi.so -NA_BMI_TESTING_PROTOCOL:STRING=tcp -NA_USE_MPI:BOOL=ON -OPA_INCLUDE_DIR:PATH=/usr/include/mpich2 -OPA_LIBRARY:FILEPATH=/usr/lib/libopa.so -NA_USE_CCI:BOOL=ON -CCI_INCLUDE_DIR:PATH=$ENV{WORKSPACE}/.install/include -CCI_LIBRARY:FILEPATH=$ENV{WORKSPACE}/.install/lib/libcci.so -NA_CCI_TESTING_PROTOCOL:STRING=tcp;sm -MPIEXEC_MAX_NUMPROCS:STRING=4 - -MERCURY_TEST_INIT_COMMAND:STRING=killall -9 -r hg_test_client;killall -9 -r hg_test_server; -MERCURY_TESTING_CORESIDENT:BOOL=ON -") - -set(dashboard_git_url $ENV{GIT_URL}) -#set(dashboard_git_branch $ENV{GIT_BRANCH}) -#set(dashboard_git_commit $ENV{GIT_COMMIT}) - -#set(ENV{CC} /usr/bin/gcc) -#set(ENV{CXX} /usr/bin/g++) - -include(mercury_common.cmake) - -####################################################################### diff --git a/Testing/script/mercury_common.cmake b/Testing/script/mercury_common.cmake index 16522411..beec5df2 100644 --- a/Testing/script/mercury_common.cmake +++ b/Testing/script/mercury_common.cmake @@ -1,11 +1,12 @@ -# Mercury Common Dashboard Script +# Common Dashboard Script # # This script contains basic dashboard driver code common to all -# clients. +# clients and projects. It is a combination of the universal.cmake script in +# the Kitware DashboardScriptsNG repo and cmake_common.cmake used by CMake +# dashboards. # -# Put this script in a directory such as "~/Dashboards/Scripts" or -# "c:/Dashboards/Scripts". Create a file next to this script, say -# 'my_dashboard.cmake', with code of the following form: +# Create a file next to this script, say 'my_dashboard.cmake', with code of the +# following form: # # # Client maintainer: me@mydomain.net # set(CTEST_SITE "machine.site") @@ -16,7 +17,7 @@ # # Then run a scheduled task (cron job) with a command line such as # -# ctest -S ~/Dashboards/Scripts/my_dashboard.cmake -V +# ctest -S my_dashboard.cmake -V # # By default the source and build trees will be placed in the path # "../My Tests/" relative to your script location. @@ -24,26 +25,35 @@ # The following variables may be set before including this script # to configure it: # -# dashboard_model = Nightly | Experimental | Continuous -# dashboard_disable_loop = For continuous dashboards, disable loop. -# dashboard_root_name = Change name of "My Tests" directory -# dashboard_source_name = Name of source directory (Mercury) -# dashboard_binary_name = Name of binary directory (Mercury-build) -# dashboard_cache = Initial CMakeCache.txt file content -# dashboard_do_coverage = True to enable coverage (ex: gcov) -# dashboard_do_memcheck = True to enable memcheck (ex: valgrind) -# CTEST_BUILD_FLAGS = build tool arguments (ex: -j2) -# CTEST_DASHBOARD_ROOT = Where to put source and build trees -# CTEST_TEST_CTEST = Whether to run long CTestTest* tests -# CTEST_TEST_TIMEOUT = Per-test timeout length -# CTEST_TEST_ARGS = ctest_test args (ex: PARALLEL_LEVEL 4) -# CMAKE_MAKE_PROGRAM = Path to "make" tool to use +# dashboard_model = Nightly | Experimental | Continuous +# dashboard_root_name = Change name of "My Tests" directory +# dashboard_source_name = Name of source directory (Mercury) +# dashboard_binary_name = Name of binary directory (Mercury-build) +# dashboard_cache = Initial CMakeCache.txt file content +# dashboard_track = The name of the CDash "Track" to submit to # -# Options to configure builds from experimental git repository: -# dashboard_git_url = Custom git clone url -# dashboard_git_branch = Custom remote branch to track -# dashboard_git_commit = Custom commit to checkout -# dashboard_git_crlf = Value of core.autocrlf for repository +# dashboard_do_checkout = True to enable source checkout via git +# dashboard_do_update = True to enable the Update step +# dashboard_do_configure = True to enable the Configure step +# dashboard_do_build = True to enable the Build step +# dashboard_do_test = True to enable the Test step +# dashboard_do_coverage = True to enable coverage (ex: gcov) +# dashboard_do_memcheck = True to enable memcheck (ex: valgrind) +# dashboard_do_submit = Submit each step (ON) +# dashboard_do_submit_only = Only submit step results, do nto run them (OFF) +# +# CTEST_GIT_COMMAND = path to git command-line client +# CTEST_BUILD_FLAGS = build tool arguments (ex: -j2) +# CTEST_DASHBOARD_ROOT = Where to put source and build trees +# CTEST_TEST_CTEST = Whether to run long CTestTest* tests +# CTEST_TEST_TIMEOUT = Per-test timeout length +# CTEST_TEST_ARGS = ctest_test args (ex: PARALLEL_LEVEL 4) +# CMAKE_MAKE_PROGRAM = Path to "make" tool to use +# +# Options to configure Git: +# dashboard_git_url = Custom git clone url +# dashboard_git_branch = Custom remote branch to track +# dashboard_git_crlf = Value of core.autocrlf for repository # # For Makefile generators the script may be executed from an # environment already configured to use the desired compilers. @@ -54,12 +64,61 @@ # set(ENV{FC} /path/to/fc) # Fortran compiler (optional) # set(ENV{LD_LIBRARY_PATH} /path/to/vendor/lib) # (if necessary) -cmake_minimum_required(VERSION 2.8 FATAL_ERROR) +cmake_minimum_required(VERSION 2.8.2 FATAL_ERROR) + +if(NOT DEFINED dashboard_full) + set(dashboard_full TRUE) +endif() -set(CTEST_PROJECT_NAME Mercury) -set(dashboard_user_home "$ENV{HOME}") +# Initialize all build steps to "ON" +if(NOT DEFINED dashboard_do_update) + set(dashboard_do_update ${dashboard_full}) +endif() + +if(NOT DEFINED dashboard_do_checkout) + set(dashboard_do_checkout ${dashboard_full}) +endif() + +if(NOT DEFINED dashboard_do_configure) + set(dashboard_do_configure ${dashboard_full}) +endif() + +if(NOT DEFINED dashboard_do_build) + set(dashboard_do_build ${dashboard_full}) +endif() + +if(NOT DEFINED dashboard_do_test) + set(dashboard_do_test ${dashboard_full}) +endif() + +# Default code coverage and memtesting to off +if(NOT DEFINED dashboard_do_coverage) + set(dashboard_do_coverage FALSE) +endif() + +if(NOT DEFINED dashboard_do_memcheck) + set(dashboard_do_memcheck FALSE) +endif() + +if(NOT DEFINED dashboard_fresh) + if(dashboard_full OR dashboard_do_update) + set(dashboard_fresh TRUE) + else() + set(dashboard_fresh FALSE) + endif() +endif() + +if(NOT DEFINED dashboard_do_submit_only) + set(dashboard_do_submit_only FALSE) +endif() + +if(NOT DEFINED dashboard_do_submit) + set(dashboard_do_submit TRUE) +endif() -get_filename_component(dashboard_self_dir ${CMAKE_CURRENT_LIST_FILE} PATH) +if(NOT DEFINED CTEST_PROJECT_NAME) + message(FATAL_ERROR "project-specific script including '***_common.cmake' should set CTEST_PROJECT_NAME") +endif() # Select the top dashboard directory. if(NOT DEFINED dashboard_root_name) @@ -78,16 +137,16 @@ if(NOT "${dashboard_model}" MATCHES "^(Nightly|Experimental|Continuous)$") endif() # Default to a Debug build. -if(NOT DEFINED CTEST_CONFIGURATION_TYPE AND DEFINED CTEST_BUILD_CONFIGURATION) - set(CTEST_CONFIGURATION_TYPE ${CTEST_BUILD_CONFIGURATION}) +if(NOT DEFINED CTEST_BUILD_CONFIGURATION) + set(CTEST_BUILD_CONFIGURATION Debug) endif() if(NOT DEFINED CTEST_CONFIGURATION_TYPE) - set(CTEST_CONFIGURATION_TYPE Debug) + set(CTEST_CONFIGURATION_TYPE ${CTEST_BUILD_CONFIGURATION}) endif() # Choose CTest reporting mode. -if(NOT "${CTEST_CMAKE_GENERATOR}" MATCHES "Make") +if(NOT "${CTEST_CMAKE_GENERATOR}" MATCHES "Make|Ninja") # Launchers work only with Makefile generators. set(CTEST_USE_LAUNCHERS 0) elseif(NOT DEFINED CTEST_USE_LAUNCHERS) @@ -103,36 +162,40 @@ if(NOT CTEST_TEST_TIMEOUT) set(CTEST_TEST_TIMEOUT 1500) endif() - # Select Git source to use. if(NOT DEFINED dashboard_git_url) - set(dashboard_git_url "git://git.mcs.anl.gov/radix/mercury") + message(FATAL_ERROR "project-specific script including '***_common.cmake' should set dashboard_git_url") endif() - if(NOT DEFINED dashboard_git_branch) set(dashboard_git_branch master) endif() - if(NOT DEFINED dashboard_git_crlf) if(UNIX) set(dashboard_git_crlf false) - else(UNIX) + else() set(dashboard_git_crlf true) - endif(UNIX) + endif() endif() # Look for a GIT command-line client. if(NOT DEFINED CTEST_GIT_COMMAND) - find_program(CTEST_GIT_COMMAND NAMES git git.cmd) + set(git_names git git.cmd) + # First search the PATH. + find_program(CTEST_GIT_COMMAND NAMES ${git_names}) + if(CMAKE_HOST_WIN32) + # Now look for installations in Git/ directories under typical installation + # prefixes on Windows. Exclude PATH from this search because VS 2017's + # command prompt happens to have a PATH entry with a Git/ subdirectory + # containing a minimal git not meant for general use. + find_program(CTEST_GIT_COMMAND + NAMES ${git_names} + PATH_SUFFIXES Git/cmd Git/bin + NO_SYSTEM_ENVIRONMENT_PATH + ) + endif() endif() - -if(NOT DEFINED CTEST_GIT_COMMAND) - message(FATAL_ERROR "No Git Found.") -endif() - -# Look for a COVERAGE command -if(NOT DEFINED CTEST_COVERAGE_COMMAND) - find_program(CTEST_COVERAGE_COMMAND NAMES gcov) +if(NOT CTEST_GIT_COMMAND) + message(FATAL_ERROR "CTEST_GIT_COMMAND not available!") endif() # Select a source directory name. @@ -140,7 +203,7 @@ if(NOT DEFINED CTEST_SOURCE_DIRECTORY) if(DEFINED dashboard_source_name) set(CTEST_SOURCE_DIRECTORY ${CTEST_DASHBOARD_ROOT}/${dashboard_source_name}) else() - set(CTEST_SOURCE_DIRECTORY ${CTEST_DASHBOARD_ROOT}/mercury) + set(CTEST_SOURCE_DIRECTORY ${CTEST_DASHBOARD_ROOT}/${CTEST_PROJECT_NAME}) endif() endif() @@ -153,40 +216,51 @@ if(NOT DEFINED CTEST_BINARY_DIRECTORY) endif() endif() -# Delete source tree if it is incompatible with current VCS. -if(EXISTS ${CTEST_SOURCE_DIRECTORY}) - if(NOT EXISTS "${CTEST_SOURCE_DIRECTORY}/.git") - set(vcs_refresh "because it is not managed by git.") - endif() - if(vcs_refresh AND "${CTEST_SOURCE_DIRECTORY}" MATCHES "/mercury[^/]*") - message("Deleting source tree\n ${CTEST_SOURCE_DIRECTORY}\n${vcs_refresh}") - file(REMOVE_RECURSE "${CTEST_SOURCE_DIRECTORY}") - endif() -endif() +macro(dashboard_git) + execute_process( + COMMAND ${CTEST_GIT_COMMAND} ${ARGN} + WORKING_DIRECTORY "${CTEST_SOURCE_DIRECTORY}" + OUTPUT_VARIABLE dashboard_git_output + ERROR_VARIABLE dashboard_git_output + RESULT_VARIABLE dashboard_git_failed + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_STRIP_TRAILING_WHITESPACE + ) +endmacro() -# Support initial checkout if necessary. -if(NOT EXISTS "${CTEST_SOURCE_DIRECTORY}" - AND NOT DEFINED CTEST_CHECKOUT_COMMAND) - get_filename_component(_name "${CTEST_SOURCE_DIRECTORY}" NAME) - execute_process(COMMAND ${CTEST_GIT_COMMAND} --version OUTPUT_VARIABLE output) - string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]+(\\.[0-9]+(\\.g[0-9a-f]+)?)?" GIT_VERSION "${output}") - if(NOT "${GIT_VERSION}" VERSION_LESS "1.6.5") - # Have "git clone -b " option. - set(git_branch_new "-b ${dashboard_git_branch}") - set(git_branch_old) - else() - # No "git clone -b " option. - set(git_branch_new) - set(git_branch_old "-b ${dashboard_git_branch} origin/${dashboard_git_branch}") +if(dashboard_do_checkout) + # Delete source tree if it is incompatible with current VCS. + if(EXISTS ${CTEST_SOURCE_DIRECTORY}) + if(NOT EXISTS "${CTEST_SOURCE_DIRECTORY}/.git") + set(vcs_refresh "because it is not managed by git.") + else() + execute_process( + COMMAND ${CTEST_GIT_COMMAND} reset --hard + WORKING_DIRECTORY "${CTEST_SOURCE_DIRECTORY}" + OUTPUT_VARIABLE output + ERROR_VARIABLE output + RESULT_VARIABLE failed + ) + if(failed) + set(vcs_refresh "because its .git may be corrupted.") + endif() + endif() + if(vcs_refresh AND "${CTEST_SOURCE_DIRECTORY}" MATCHES "/CMake[^/]*") + message("Deleting source tree\n") + message(" ${CTEST_SOURCE_DIRECTORY}\n${vcs_refresh}") + file(REMOVE_RECURSE "${CTEST_SOURCE_DIRECTORY}") + endif() endif() + # Support initial checkout if necessary. + if(NOT EXISTS "${CTEST_SOURCE_DIRECTORY}" + AND NOT DEFINED CTEST_CHECKOUT_COMMAND) # Generate an initial checkout script. + get_filename_component(_name "${CTEST_SOURCE_DIRECTORY}" NAME) set(ctest_checkout_script ${CTEST_DASHBOARD_ROOT}/${_name}-init.cmake) - -############################## File ####################################### - file(WRITE ${ctest_checkout_script} "# git repo init script for ${_name} + file(WRITE ${ctest_checkout_script} "# git repo init script for ${_name} execute_process( - COMMAND \"${CTEST_GIT_COMMAND}\" clone ${git_branch_new} -- \"${dashboard_git_url}\" + COMMAND \"${CTEST_GIT_COMMAND}\" clone -n -- \"${dashboard_git_url}\" \"${CTEST_SOURCE_DIRECTORY}\" ) if(EXISTS \"${CTEST_SOURCE_DIRECTORY}/.git\") @@ -195,39 +269,40 @@ if(EXISTS \"${CTEST_SOURCE_DIRECTORY}/.git\") WORKING_DIRECTORY \"${CTEST_SOURCE_DIRECTORY}\" ) execute_process( - COMMAND \"${CTEST_GIT_COMMAND}\" checkout ${git_branch_old} + COMMAND \"${CTEST_GIT_COMMAND}\" fetch WORKING_DIRECTORY \"${CTEST_SOURCE_DIRECTORY}\" ) execute_process( - COMMAND \"${CTEST_GIT_COMMAND}\" submodule init + COMMAND \"${CTEST_GIT_COMMAND}\" checkout ${dashboard_git_branch} WORKING_DIRECTORY \"${CTEST_SOURCE_DIRECTORY}\" ) execute_process( - COMMAND \"${CTEST_GIT_COMMAND}\" submodule update -- + COMMAND \"${CTEST_GIT_COMMAND}\" submodule update --init WORKING_DIRECTORY \"${CTEST_SOURCE_DIRECTORY}\" ) -endif() -") -############################## File ####################################### - +endif()" + ) set(CTEST_CHECKOUT_COMMAND "\"${CMAKE_COMMAND}\" -P \"${ctest_checkout_script}\"") - # CTest delayed initialization is broken, so we put the - # CTestConfig.cmake info here. - set(CTEST_NIGHTLY_START_TIME "00:00:00 CST") - set(CTEST_DROP_METHOD "http") - set(CTEST_DROP_SITE "cdash.hdfgroup.org") - set(CTEST_DROP_LOCATION "/submit.php?project=MERCURY") - set(CTEST_DROP_SITE_CDASH TRUE) + elseif(EXISTS "${CTEST_SOURCE_DIRECTORY}/.git") + # Upstream URL. + dashboard_git(config --get remote.origin.url) + if(NOT dashboard_git_output STREQUAL "${dashboard_git_url}") + dashboard_git(config remote.origin.url "${dashboard_git_url}") + endif() + + # Local checkout. + dashboard_git(symbolic-ref HEAD) + if(NOT dashboard_git_output STREQUAL "${dashboard_git_branch}") + dashboard_git(checkout --recurse-submodules ${dashboard_git_branch}) + if(dashboard_git_failed) + message(FATAL_ERROR "Failed to checkout branch ${dashboard_git_branch}:\n${dashboard_git_output}") + endif() + endif() + endif() endif() #----------------------------------------------------------------------------- -# Send the main script as a note. -list(APPEND CTEST_NOTES_FILES - "${CTEST_SCRIPT_DIRECTORY}/${CTEST_SCRIPT_NAME}" - "${CMAKE_CURRENT_LIST_FILE}" - ) - # Check for required variables. foreach(req CTEST_CMAKE_GENERATOR @@ -240,6 +315,7 @@ foreach(req endforeach(req) # Print summary information. +set(vars "") foreach(v CTEST_SITE CTEST_BUILD_NAME @@ -249,6 +325,7 @@ foreach(v CTEST_BUILD_CONFIGURATION CTEST_GIT_COMMAND CTEST_CHECKOUT_COMMAND + CTEST_CONFIGURE_COMMAND CTEST_SCRIPT_DIRECTORY CTEST_USE_LAUNCHERS ) @@ -263,7 +340,7 @@ set(ENV{LC_ALL} C) macro(write_cache) set(cache_build_type "") set(cache_make_program "") - if(CTEST_CMAKE_GENERATOR MATCHES "Make") + if(CTEST_CMAKE_GENERATOR MATCHES "Make|Ninja") set(cache_build_type CMAKE_BUILD_TYPE:STRING=${CTEST_BUILD_CONFIGURATION}) if(CMAKE_MAKE_PROGRAM) set(cache_make_program CMAKE_MAKE_PROGRAM:FILEPATH=${CMAKE_MAKE_PROGRAM}) @@ -272,101 +349,166 @@ macro(write_cache) file(WRITE ${CTEST_BINARY_DIRECTORY}/CMakeCache.txt " SITE:STRING=${CTEST_SITE} BUILDNAME:STRING=${CTEST_BUILD_NAME} +CTEST_TEST_CTEST:BOOL=${CTEST_TEST_CTEST} CTEST_USE_LAUNCHERS:BOOL=${CTEST_USE_LAUNCHERS} DART_TESTING_TIMEOUT:STRING=${CTEST_TEST_TIMEOUT} +GIT_EXECUTABLE:FILEPATH=${CTEST_GIT_COMMAND} ${cache_build_type} ${cache_make_program} ${dashboard_cache} ") endmacro(write_cache) -# Start with a fresh build tree. -file(MAKE_DIRECTORY "${CTEST_BINARY_DIRECTORY}") -if(NOT "${CTEST_SOURCE_DIRECTORY}" STREQUAL "${CTEST_BINARY_DIRECTORY}") - file(GLOB CTEST_BINARY_DIRECTORY_LIST ${CTEST_BINARY_DIRECTORY}/*.txt) - list(LENGTH CTEST_BINARY_DIRECTORY_LIST CTEST_BINARY_DIRECTORY_LIST_LEN) - if(NOT CTEST_BINARY_DIRECTORY_LIST_LEN EQUAL 0) - message("Clearing build tree...") - ctest_empty_binary_directory(${CTEST_BINARY_DIRECTORY}) +if(COMMAND dashboard_hook_init) + dashboard_hook_init() +endif() + +if(dashboard_fresh) + # Start with a fresh build tree. + if(EXISTS "${CTEST_BINARY_DIRECTORY}" AND + NOT "${CTEST_SOURCE_DIRECTORY}" STREQUAL "${CTEST_BINARY_DIRECTORY}") + message("Clearing build tree...") + ctest_empty_binary_directory(${CTEST_BINARY_DIRECTORY}) endif() + file(MAKE_DIRECTORY "${CTEST_BINARY_DIRECTORY}") + message("Starting fresh build...") + write_cache() endif() -set(dashboard_continuous 0) -if("${dashboard_model}" STREQUAL "Continuous") - set(dashboard_continuous 1) +# Start a new submission. +if(dashboard_track) + set(dashboard_track_arg TRACK "${dashboard_track}") endif() -if (dashboard_continous_force) - set(dashboard_continuous 1) +message("Calling ctest_start") +if(dashboard_fresh) + if(COMMAND dashboard_hook_start) + dashboard_hook_start() + endif() + ctest_start(${dashboard_model} ${dashboard_track_arg}) + if(dashboard_do_submit) + ctest_submit(PARTS Start) + endif() + if(COMMAND dashboard_hook_started) + dashboard_hook_started() + endif() +else() + ctest_start(${dashboard_model} ${dashboard_track_arg} APPEND) endif() -# CTest 2.6 crashes with message() after ctest_test. -macro(safe_message) - if(NOT "${CMAKE_VERSION}" VERSION_LESS 2.8 OR NOT safe_message_skip) - message(${ARGN}) - endif() -endmacro() +# Look for updates. +if(NOT dashboard_do_update) + set(CTEST_UPDATE_VERSION_ONLY TRUE) +endif() -set(dashboard_done 0) -while(NOT dashboard_done) - if(dashboard_continuous) - set(START_TIME ${CTEST_ELAPSED_TIME}) +if(NOT dashboard_do_submit_only) + if(COMMAND dashboard_hook_update) + dashboard_hook_update() endif() - set(ENV{HOME} "${dashboard_user_home}") + message("Calling ctest_update...") + ctest_update(RETURN_VALUE count) + set(CTEST_CHECKOUT_COMMAND) # checkout on first iteration only + message("Found ${count} changed files") +endif() - # Start a new submission. - ctest_start(${dashboard_model}) +if(dashboard_do_submit) + if(CTEST_SUBMIT_NOTES) + message("Submitting dashboard scripts as Notes") + # Send the main script as a note while submitting the Update part + set(CTEST_NOTES_FILES + ${CTEST_UPDATE_NOTES_FILES} + "${CMAKE_CURRENT_LIST_FILE}") + ctest_submit(PARTS Update Notes) + unset(CTEST_NOTES_FILES) + else() + message("Skipping notes submission for Update step") + ctest_submit(PARTS Update) + endif() +endif() - # Always build if the tree is fresh. - set(dashboard_fresh 0) - if(NOT EXISTS "${CTEST_BINARY_DIRECTORY}/CMakeCache.txt") - set(dashboard_fresh 1) - safe_message("Starting fresh build...") - write_cache() +if(dashboard_do_configure) + if(NOT dashboard_do_submit_only) + if(COMMAND dashboard_hook_configure) + dashboard_hook_configure() + endif() + message("Calling ctest_configure)") + ctest_configure(${dashboard_configure_args}) endif() - - # Look for updates. - ctest_update(SOURCE ${CTEST_SOURCE_DIRECTORY} - RETURN_VALUE count) - safe_message("Found ${count} changed files") - - # get newest submodule info - execute_process( - COMMAND "${CTEST_GIT_COMMAND}" submodule update --init - WORKING_DIRECTORY "${CTEST_SOURCE_DIRECTORY}" - ) - - if(dashboard_fresh OR NOT dashboard_continuous OR count GREATER 0) - ctest_configure() - ctest_submit(PARTS Update Configure Notes) - ctest_read_custom_files(${CTEST_BINARY_DIRECTORY}) + if(dashboard_do_submit) + if(CTEST_SUBMIT_NOTES) + message("Submitting CMakeCache.txt as Notes") + set(CTEST_NOTES_FILES "${CTEST_BINARY_DIRECTORY}/CMakeCache.txt") + ctest_submit(PARTS Configure Notes) + unset(CTEST_NOTES_FILES) + else() + message("Skipping notes submission for Configure step") + ctest_submit(PARTS Configure) + endif() + endif() +endif() - ctest_build(APPEND) +ctest_read_custom_files(${CTEST_BINARY_DIRECTORY}) + +if(dashboard_do_build) + if(NOT dashboard_do_submit_only) + if(COMMAND dashboard_hook_build) + dashboard_hook_build() + endif() + message("Calling ctest_build") + ctest_build( + NUMBER_WARNINGS ctest_build_num_warnings + ) + endif() + if(dashboard_do_submit) ctest_submit(PARTS Build) + endif() +endif() -# ctest_test(INCLUDE_LABEL "mercury" ${CTEST_TEST_ARGS} APPEND) - ctest_test(${CTEST_TEST_ARGS} APPEND) +if(dashboard_do_test) + if(NOT dashboard_do_submit_only) + if(COMMAND dashboard_hook_test) + dashboard_hook_test() + endif() + message("Calling ctest_test") + ctest_test(${CTEST_TEST_ARGS} RETURN_VALUE TEST_RESULTS) + if(${TEST_RESULTS} EQUAL 0) + message("ctest test results return value: ${TEST_RESULTS}") + else() + #message(SEND_ERROR "Some tests failed") + message("Some tests failed") + endif() + endif() + if(dashboard_do_submit) ctest_submit(PARTS Test) - set(safe_message_skip 1) # Block further messages + endif() +endif() - if(dashboard_do_coverage) - ctest_coverage() - ctest_submit(PARTS Coverage) - endif() - if(dashboard_do_memcheck) - ctest_memcheck() - ctest_submit(PARTS MemCheck) +if(dashboard_do_coverage) + if(NOT dashboard_do_submit_only) + if(COMMAND dashboard_hook_coverage) + dashboard_hook_coverage() endif() + message("Calling ctest_coverage") + ctest_coverage() + endif() + if(dashboard_do_submit) + ctest_submit(PARTS Coverage) endif() +endif() - if(dashboard_continuous AND NOT dashboard_disable_loop) - # Delay until at least 5 minutes past START_TIME - ctest_sleep(${START_TIME} 300 ${CTEST_ELAPSED_TIME}) - if(${CTEST_ELAPSED_TIME} GREATER 57600) - set(dashboard_done 1) +if(dashboard_do_memcheck) + if(NOT dashboard_do_submit_only) + if(COMMAND dashboard_hook_memcheck) + dashboard_hook_memcheck() endif() - else() - # Not continuous, so we are done. - set(dashboard_done 1) + message("Calling ctest_memcheck") + ctest_memcheck() + endif() + if(dashboard_do_submit) + ctest_submit(PARTS MemCheck) endif() -endwhile() +endif() + +if(COMMAND dashboard_hook_end) + dashboard_hook_end() +endif() diff --git a/Testing/script/travis_script.cmake b/Testing/script/travis_script.cmake index 391f21a8..c981e557 100644 --- a/Testing/script/travis_script.cmake +++ b/Testing/script/travis_script.cmake @@ -1,11 +1,40 @@ # This script takes in optional environment variables. -# MERCURY_BUILD_CONFIGURATION=Debug | Release +# MERCURY_BUILD_CONFIGURATION=Debug | RelWithDebInfo | Release | Asan | Tsan # MERCURY_DASHBOARD_MODEL=Experimental | Nightly | Continuous # MERCURY_BUILD_STATIC_LIBRARIES # MERCURY_DO_COVERAGE # MERCURY_DO_MEMCHECK -# MERCURY_BUILD_CONFIGURATION = Debug | Release +set(CTEST_PROJECT_NAME "MERCURY") + +if(NOT dashboard_git_url) + set(dashboard_git_url "https://github.com/mercury-hpc/mercury.git") +endif() + +# Checkout is done by travis +set(dashboard_do_checkout 0) +set(dashboard_do_update 0) + +if(NOT DEFINED CTEST_TEST_TIMEOUT) + set(CTEST_TEST_TIMEOUT 180) +endif() + +if(NOT DEFINED CTEST_SUBMIT_NOTES) + set(CTEST_SUBMIT_NOTES TRUE) +endif() + +# Give a site name +set(CTEST_SITE "worker.travis-ci.org") + +# Must specify existing source directory +set(CTEST_SOURCE_DIRECTORY "$ENV{TRAVIS_BUILD_DIR}") +set(CTEST_BINARY_DIRECTORY "${CTEST_SOURCE_DIRECTORY}/build") + +set(OS_NAME "$ENV{TRAVIS_OS_NAME}") + +set(BUILD_NUMBER "$ENV{TRAVIS_BUILD_NUMBER}") + +# MERCURY_BUILD_CONFIGURATION set(MERCURY_BUILD_CONFIGURATION "$ENV{MERCURY_BUILD_CONFIGURATION}") if(NOT MERCURY_BUILD_CONFIGURATION) set(MERCURY_BUILD_CONFIGURATION "Debug") @@ -13,6 +42,13 @@ endif() string(TOLOWER ${MERCURY_BUILD_CONFIGURATION} lower_mercury_build_configuration) set(CTEST_BUILD_CONFIGURATION ${MERCURY_BUILD_CONFIGURATION}) +if(MERCURY_BUILD_CONFIGURATION MATCHES "Asan") + set(MERCURY_MEMORYCHECK_TYPE "AddressSanitizer") +endif() +if(MERCURY_BUILD_CONFIGURATION MATCHES "Tsan") + set(MERCURY_MEMORYCHECK_TYPE "ThreadSanitizer") +endif() + # MERCURY_DASHBOARD_MODEL=Experimental | Nightly | Continuous set(MERCURY_DASHBOARD_MODEL "$ENV{MERCURY_DASHBOARD_MODEL}") if(NOT MERCURY_DASHBOARD_MODEL) @@ -20,18 +56,11 @@ if(NOT MERCURY_DASHBOARD_MODEL) endif() set(dashboard_model ${MERCURY_DASHBOARD_MODEL}) -# Disable loop when MERCURY_DASHBOARD_MODEL=Continuous -set(MERCURY_NO_LOOP $ENV{MERCURY_NO_LOOP}) -if(MERCURY_NO_LOOP) - message("Disabling looping (if applicable)") - set(dashboard_disable_loop TRUE) -endif() - -# Disable source tree update and use current version -set(CTEST_UPDATE_VERSION_ONLY TRUE) +# Add current script to notes files +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") -# Number of jobs to build and verbose mode -set(CTEST_BUILD_FLAGS "-j4") +# Number of jobs to build and keep going if some targets can't be made +set(CTEST_BUILD_FLAGS "-k -j4") # Build shared libraries set(mercury_build_shared ON) @@ -42,19 +71,13 @@ if(MERCURY_BUILD_STATIC_LIBRARIES) endif() set(CTEST_CMAKE_GENERATOR "Unix Makefiles") -# Must point to the root where we can checkout/build/run the tests -set(CTEST_DASHBOARD_ROOT "$ENV{TRAVIS_BUILD_DIR}/..") -# Must specify existing source directory -set(CTEST_SOURCE_DIRECTORY "$ENV{TRAVIS_BUILD_DIR}") -# Give a site name -set(CTEST_SITE "worker.travis-ci.org") -set(CTEST_TEST_TIMEOUT 180) # 180s timeout # Optional coverage options set(MERCURY_DO_COVERAGE $ENV{MERCURY_DO_COVERAGE}) if(MERCURY_DO_COVERAGE) message("Enabling Coverage") set(CTEST_COVERAGE_COMMAND "/usr/bin/$ENV{COV}") + # don't run parallel coverage tests, no matter what. set(CTEST_TEST_ARGS PARALLEL_LEVEL 1) @@ -63,22 +86,16 @@ if(MERCURY_DO_COVERAGE) # build suffix set(coverage_suffix "-coverage") - - # add Coverage dir to the root so that we don't mess the non-coverage - # dashboard. - set(CTEST_DASHBOARD_ROOT "${CTEST_DASHBOARD_ROOT}/Coverage") endif() # Optional memcheck options set(MERCURY_DO_MEMCHECK $ENV{MERCURY_DO_MEMCHECK}) -set(MERCURY_MEMORYCHECK_TYPE "$ENV{MERCURY_MEMORYCHECK_TYPE}") if(MERCURY_DO_MEMCHECK OR MERCURY_MEMORYCHECK_TYPE) message("Enabling Memcheck") if(NOT MERCURY_MEMORYCHECK_TYPE) set(MERCURY_MEMORYCHECK_TYPE "Valgrind") endif() - string(TOLOWER "-${MERCURY_MEMORYCHECK_TYPE}" lower_mercury_memorycheck_type) set(CTEST_MEMORYCHECK_TYPE ${MERCURY_MEMORYCHECK_TYPE}) # Valgrind @@ -87,15 +104,15 @@ if(MERCURY_DO_MEMCHECK OR MERCURY_MEMORYCHECK_TYPE) set(CTEST_MEMORYCHECK_COMMAND_OPTIONS "--gen-suppressions=all --trace-children=yes --fair-sched=yes -q --leak-check=yes --show-reachable=yes --num-callers=50 -v") #set(CTEST_MEMORYCHECK_SUPPRESSIONS_FILE ${CTEST_SCRIPT_DIRECTORY}/MercuryValgrindSuppressions.supp) endif() + # Tsan if(${MERCURY_MEMORYCHECK_TYPE} MATCHES "ThreadSanitizer") - set(MERCURY_MEMCHECK_FLAGS "-O1 -fsanitize=thread -fno-omit-frame-pointer -fPIC -fuse-ld=gold -pthread") # Must add verbosity / Error in build if no memory output file is produced set(CTEST_MEMORYCHECK_SANITIZER_OPTIONS "verbosity=1") endif() + # Asan if(${MERCURY_MEMORYCHECK_TYPE} MATCHES "AddressSanitizer") - set(MERCURY_MEMCHECK_FLAGS "-O1 -fsanitize=address -fno-omit-frame-pointer -fPIC -fuse-ld=gold -pthread") # Must add verbosity / Error in build if no memory output file is produced set(CTEST_MEMORYCHECK_SANITIZER_OPTIONS "verbosity=1") endif() @@ -105,7 +122,7 @@ if(MERCURY_DO_MEMCHECK OR MERCURY_MEMORYCHECK_TYPE) endif() # Build name referenced in cdash -set(CTEST_BUILD_NAME "travis-ci-$ENV{TRAVIS_OS_NAME}-x64-$ENV{CC}-${lower_mercury_build_configuration}${lower_mercury_memorycheck_type}${coverage_suffix}-$ENV{TRAVIS_BUILD_NUMBER}") +set(CTEST_BUILD_NAME "travis-ci-${OS_NAME}-x64-$ENV{CC}-${lower_mercury_build_configuration}${coverage_suffix}-${BUILD_NUMBER}") set(dashboard_binary_name mercury-${lower_mercury_build_configuration}) if(NOT mercury_build_shared) @@ -129,13 +146,15 @@ else() endif() if($ENV{CC} MATCHES "^gcc.*") - set(MERCURY_C_FLAGS "-Wall -Wextra -Wshadow -Winline -Wundef -Wcast-qual -Wconversion -Wmissing-prototypes -pedantic -Wpointer-arith -Wformat=2 -std=gnu11 ${MERCURY_MEMCHECK_FLAGS}") + set(MERCURY_C_FLAGS "-Wall -Wextra -Wshadow -Winline -Wundef -Wcast-qual -Wconversion -Wmissing-prototypes -pedantic -Wpointer-arith -Wformat=2 -std=gnu11") endif() +set(MERCURY_C_FLAGS ${MERCURY_C_FLAGS}) +set(MERCURY_CXX_FLAGS ${MERCURY_CXX_FLAGS}) # Initial cache used to build mercury, options can be modified here set(dashboard_cache " CMAKE_C_FLAGS:STRING=${MERCURY_C_FLAGS} -CMAKE_CXX_FLAGS:STRING=${MERCURY_MEMCHECK_FLAGS} +CMAKE_CXX_FLAGS:STRING=${MERCURY_CXX_FLAGS} BUILD_SHARED_LIBS:BOOL=${mercury_build_shared} BUILD_TESTING:BOOL=ON @@ -165,9 +184,4 @@ MERCURY_TESTING_MAX_HANDLES:STRING=1 MERCURY_TESTING_CORESIDENT:BOOL=ON ") -#set(ENV{CC} /usr/bin/gcc) -#set(ENV{CXX} /usr/bin/g++) - include(${CTEST_SOURCE_DIRECTORY}/Testing/script/mercury_common.cmake) - -####################################################################### From 7a2a27387de90b5ddb1acbb6900e2f4115c7081e Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Thu, 4 Jun 2020 16:33:08 -0500 Subject: [PATCH 14/30] Update travis build for OFI 1.10.1 and cmake 3.17.3 --- Testing/script/travis_build.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Testing/script/travis_build.sh b/Testing/script/travis_build.sh index 69997f21..c6cfd28a 100755 --- a/Testing/script/travis_build.sh +++ b/Testing/script/travis_build.sh @@ -1,10 +1,10 @@ #!/bin/bash BMI_VERSION=master -CMAKE_VERSION_MAJOR=3.16 -CMAKE_VERSION_MINOR=5 +CMAKE_VERSION_MAJOR=3.17 +CMAKE_VERSION_MINOR=3 MPI_VERSION=3.3.2 -OFI_VERSION=1.9.1 +OFI_VERSION=1.10.1 PREFIX=$HOME/install set -e @@ -58,7 +58,7 @@ if [[ $TRAVIS_OS_NAME == 'linux' ]]; then if [ ! -f "$PREFIX/bin/fi_info" ] || [ "${OFI_INSTALLED_VERSION}" != "${OFI_VERSION}" ]; then cd $HOME && wget https://github.com/ofiwg/libfabric/releases/download/v${OFI_VERSION}/libfabric-${OFI_VERSION}.tar.bz2 tar -xjf libfabric-${OFI_VERSION}.tar.bz2; - cd libfabric-${OFI_VERSION} && ./configure --prefix=$PREFIX --disable-rxd --disable-usnic --disable-mrail --disable-rstream --disable-perf --disable-efa --disable-psm2 --disable-psm --disable-udp --disable-verbs --disable-shm --disable-static --disable-silent-rules CFLAGS="-O2 -g" && make -j2 -s && make install; + cd libfabric-${OFI_VERSION} && ./configure --prefix=$PREFIX --disable-usnic --disable-mrail --disable-rstream --disable-perf --disable-efa --disable-psm2 --disable-psm --disable-verbs --disable-shm --disable-static --disable-silent-rules CFLAGS="-O2 -g" && make -j2 -s && make install; echo "${OFI_VERSION}" > $PREFIX/ofi_version.txt else echo "Using cached directory for OFI"; From 1b7910de3b6446a0265f8244ffa93e6d8ceb15c2 Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Mon, 8 Jun 2020 16:42:42 -0500 Subject: [PATCH 15/30] Travis: Force use of head for PRs instead of merge commit --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 49dc2974..9e1dd5d3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -63,6 +63,7 @@ branches: - master before_install: + - if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then git fetch origin pull/${TRAVIS_PULL_REQUEST}/head:pr${TRAVIS_PULL_REQUEST}; git checkout pr${TRAVIS_PULL_REQUEST}; fi - echo "Existing directories in $HOME" && ls $HOME - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew cask uninstall --force oclint; fi From ab66402952c33481e7c10e246492f243903b5a25 Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Mon, 8 Jun 2020 18:09:16 -0500 Subject: [PATCH 16/30] CDash: mark build as done --- Testing/script/mercury_common.cmake | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Testing/script/mercury_common.cmake b/Testing/script/mercury_common.cmake index beec5df2..cf97b6ac 100644 --- a/Testing/script/mercury_common.cmake +++ b/Testing/script/mercury_common.cmake @@ -511,4 +511,6 @@ endif() if(COMMAND dashboard_hook_end) dashboard_hook_end() endif() - +if(dashboard_do_submit) + ctest_submit(PARTS Done) +endif() From 224418b0ad205768553f93706d511199714cc171 Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Wed, 10 Jun 2020 17:17:54 -0500 Subject: [PATCH 17/30] SM: add SM header to query host ID Move UUID from HG Core to NA SM to keep SM paths in sync --- src/CMakeLists.txt | 45 ++++-------- src/mercury_core.c | 90 ++++++++--------------- src/na/CMakeLists.txt | 163 ++++++++++++++++++++---------------------- src/na/na_sm.c | 97 ++++++++++++++++++++++--- src/na/na_sm.h | 93 ++++++++++++++++++++++++ 5 files changed, 301 insertions(+), 187 deletions(-) create mode 100644 src/na/na_sm.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a2a6281f..aae3eacf 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -47,16 +47,12 @@ if(MERCURY_USE_BOOST_PP) option(MERCURY_USE_SYSTEM_BOOST "Use system-installed Boost libraries." OFF) if(MERCURY_USE_SYSTEM_BOOST) find_package(Boost 1.41.0 REQUIRED) - if(Boost_FOUND) - message(STATUS "Boost include directory: ${Boost_INCLUDE_DIRS}") - set(HG_HAS_BOOST 1) - set(MERCURY_EXT_INCLUDE_DEPENDENCIES - ${MERCURY_EXT_INCLUDE_DEPENDENCIES} - ${Boost_INCLUDE_DIRS} - ) - else() - message(FATAL_ERROR "Could not find Boost, please check Boost_INCLUDE_DIR.") - endif() + message(STATUS "Boost include directory: ${Boost_INCLUDE_DIRS}") + set(HG_HAS_BOOST 1) + set(MERCURY_EXT_INCLUDE_DEPENDENCIES + ${MERCURY_EXT_INCLUDE_DEPENDENCIES} + ${Boost_INCLUDE_DIRS} + ) else() set(HG_HAS_BOOST 1) set(MERCURY_BUILD_INCLUDE_DEPENDENCIES @@ -73,14 +69,10 @@ if(MERCURY_USE_CHECKSUMS) option(MERCURY_USE_SYSTEM_MCHECKSUM "Use system-installed MChecksum." OFF) if(MERCURY_USE_SYSTEM_MCHECKSUM) find_package(mchecksum REQUIRED) - if(mchecksum_FOUND) - set(MERCURY_EXT_PKG_DEPENDENCIES - ${MERCURY_EXT_PKG_DEPENDENCIES} - mchecksum - ) - else() - message(FATAL_ERROR "Could not find MChecksum, please check mchecksum_INCLUDE_DIR.") - endif() + set(MERCURY_EXT_PKG_DEPENDENCIES + ${MERCURY_EXT_PKG_DEPENDENCIES} + mchecksum + ) else() set(MCHECKSUM_EXTERNALLY_CONFIGURED 1) set(MCHECKSUM_INSTALL_BIN_DIR ${MERCURY_INSTALL_BIN_DIR}) @@ -129,24 +121,11 @@ option(MERCURY_USE_SM_ROUTING "Enable routing of local communication through shared-memory (requires SM)." OFF) if(MERCURY_USE_SM_ROUTING) - find_package(UUID) - if(UUID_FOUND) - set(HG_HAS_SM_ROUTING 1) - set(MERCURY_EXT_INCLUDE_DEPENDENCIES - ${MERCURY_EXT_INCLUDE_DEPENDENCIES} - ${UUID_INCLUDE_DIRS} - ) - set(MERCURY_EXT_LIB_DEPENDENCIES - ${MERCURY_EXT_LIB_DEPENDENCIES} - ${UUID_LIBRARIES} - ) - else() - message(FATAL_ERROR "Could not find libUUID, please check UUID_INCLUDE_DIR.") - endif() - if(MERCURY_USE_SM_ROUTING AND NOT NA_USE_SM) + if(NOT NA_USE_SM) # Always force SM if NA_USE_SM is turned OFF set(NA_USE_SM "ON" CACHE BOOL "Use shared-memory plugin." FORCE) endif() + set(HG_HAS_SM_ROUTING 1) endif() # Collect statistics diff --git a/src/mercury_core.c b/src/mercury_core.c index 15f45b8a..a259fc18 100644 --- a/src/mercury_core.c +++ b/src/mercury_core.c @@ -28,7 +28,7 @@ #include "mercury_error.h" #ifdef HG_HAS_SM_ROUTING -#include +#include #endif #include @@ -44,7 +44,6 @@ #define HG_CORE_MAX_EVENTS 16 #define HG_CORE_MAX_TRIGGER_COUNT 1 #ifdef HG_HAS_SM_ROUTING -# define HG_CORE_UUID_MAX_LEN 36 # define HG_CORE_ADDR_MAX_SIZE 256 # define HG_CORE_PROTO_DELIMITER ":" # define HG_CORE_ADDR_DELIMITER "#" @@ -90,7 +89,7 @@ typedef hg_atomic_int32_t hg_core_stat_t; struct hg_core_private_class { struct hg_core_class core_class; /* Must remain as first field */ #ifdef HG_HAS_SM_ROUTING - uuid_t na_sm_uuid; /* UUID for local identification */ + na_sm_id_t host_id; /* Host ID for local identification */ #endif hg_hash_table_t *func_map; /* Function map */ hg_return_t (*more_data_acquire)(hg_core_handle_t, hg_op_t, @@ -150,7 +149,7 @@ struct hg_core_self_cb_info { struct hg_core_private_addr { struct hg_core_addr core_addr; /* Must remain as first field */ #ifdef HG_HAS_SM_ROUTING - uuid_t na_sm_uuid; /* NA SM UUID */ + na_sm_id_t host_id; /* NA SM Host ID */ #endif hg_atomic_int32_t ref_count; /* Reference count */ hg_bool_t is_mine; /* Created internally or not */ @@ -229,16 +228,6 @@ struct hg_core_op_id { /* Local Prototypes */ /********************/ -#ifdef HG_HAS_SM_ROUTING -/** - * Get local ID used for detecting local nodes. - */ -static hg_return_t -hg_core_get_sm_uuid( - uuid_t *sm_uuid - ); -#endif - /** * Equal function for function map. */ @@ -783,40 +772,6 @@ hg_core_print_stats(void) } #endif -/*---------------------------------------------------------------------------*/ -#ifdef HG_HAS_SM_ROUTING -static hg_return_t -hg_core_get_sm_uuid(uuid_t *sm_uuid) -{ - const char *sm_path = NA_SM_TMP_DIRECTORY "/" NA_SM_SHM_PREFIX "/uuid.cfg"; - char uuid_str[HG_CORE_UUID_MAX_LEN + 1]; - FILE *uuid_config; - uuid_t new_uuid; - hg_return_t ret = HG_SUCCESS; - - uuid_config = fopen(sm_path, "r"); - if (!uuid_config) { - /* Generate a new one */ - uuid_generate(new_uuid); - - uuid_config = fopen(sm_path, "w"); - HG_CHECK_ERROR(uuid_config == NULL, done, ret, HG_NOENTRY, - "Could not open %s for write", sm_path); - uuid_unparse(new_uuid, uuid_str); - fprintf(uuid_config, "%s\n", uuid_str); - } else { - /* Get the existing one */ - fgets(uuid_str, HG_CORE_UUID_MAX_LEN + 1, uuid_config); - uuid_parse(uuid_str, new_uuid); - } - fclose(uuid_config); - uuid_copy(*sm_uuid, new_uuid); - -done: - return ret; -} -#endif - /*---------------------------------------------------------------------------*/ static HG_INLINE int hg_core_int_equal(void *vlocation1, void *vlocation2) @@ -1060,6 +1015,8 @@ hg_core_init(const char *na_info_string, hg_bool_t na_listen, #ifdef HG_HAS_SM_ROUTING /* Initialize SM plugin */ if (auto_sm) { + na_return_t na_ret; + HG_CHECK_ERROR(strcmp(NA_Get_class_name( hg_core_class->core_class.na_class), "na") == 0, error, ret, HG_PROTONOSUPPORT, "Cannot use auto SM mode if initialized " @@ -1071,9 +1028,10 @@ hg_core_init(const char *na_info_string, hg_bool_t na_listen, HG_CHECK_ERROR(hg_core_class->core_class.na_sm_class == NULL, error, ret, HG_NA_ERROR, "Could not initialize NA SM class"); - /* Get SM UUID */ - ret = hg_core_get_sm_uuid(&hg_core_class->na_sm_uuid); - HG_CHECK_HG_ERROR(error, ret, "Could not get SM UUID"); + /* Get SM host ID */ + na_ret = NA_SM_Host_id_get(hg_core_class->host_id); + HG_CHECK_ERROR(na_ret != NA_SUCCESS, error, ret, (hg_return_t) na_ret, + "NA_SM_Host_id_get() failed (%s)", NA_Error_to_string(na_ret)); } #endif @@ -1234,23 +1192,27 @@ hg_core_addr_lookup(struct hg_core_private_class *hg_core_class, strcpy(lookup_name, name); - /* Get first part of address string with UUID */ + /* Get first part of address string with host ID */ strtok_r(lookup_name, HG_CORE_ADDR_DELIMITER, &lookup_names); HG_CHECK_ERROR(strstr(name, HG_CORE_PROTO_DELIMITER) == NULL, error, ret, HG_PROTOCOL_ERROR, "Malformed address format"); - /* Get address SM UUID */ + /* Get address SM host ID */ strtok_r(lookup_name, HG_CORE_PROTO_DELIMITER, &local_id_str); - uuid_parse(local_id_str + 2, hg_core_addr->na_sm_uuid); + na_ret = NA_SM_String_to_host_id( + local_id_str + 2, hg_core_addr->host_id); + HG_CHECK_ERROR(na_ret != NA_SUCCESS, error, ret, (hg_return_t) na_ret, + "NA_SM_String_to_host_id() failed (%s)", + NA_Error_to_string(na_ret)); /* Separate remaining two parts */ strtok_r(lookup_names, HG_CORE_ADDR_DELIMITER, &remote_name); local_name = lookup_names; - /* Compare UUIDs, if they match it's local address */ - if (hg_core_class->core_class.na_sm_class && uuid_compare( - hg_core_addr->na_sm_uuid, hg_core_class->na_sm_uuid) == 0) { + /* Compare IDs, if they match it's local address */ + if (hg_core_class->core_class.na_sm_class && NA_SM_Host_id_cmp( + hg_core_addr->host_id, hg_core_class->host_id) == 0) { name_str = local_name; na_class = hg_core_class->core_class.na_sm_class; } else { @@ -1346,8 +1308,8 @@ hg_core_addr_self(struct hg_core_private_class *hg_core_class, HG_CHECK_ERROR(na_ret != NA_SUCCESS, done, ret, (hg_return_t) na_ret, "Could not get self SM address (%s)", NA_Error_to_string(na_ret)); - /* Copy local UUID */ - uuid_copy(hg_core_addr->na_sm_uuid, hg_core_class->na_sm_uuid); + /* Copy local host ID */ + NA_SM_Host_id_copy(hg_core_addr->host_id, hg_core_class->host_id); } #endif @@ -1409,11 +1371,15 @@ hg_core_addr_to_string(struct hg_core_private_class *hg_core_class, char *buf, #ifdef HG_HAS_SM_ROUTING if (hg_core_addr->core_addr.na_sm_addr) { char addr_str[HG_CORE_ADDR_MAX_SIZE]; - char uuid_str[HG_CORE_UUID_MAX_LEN + 1]; + char uuid_str[NA_SM_HOST_ID_LEN + 1]; int desc_len; - /* Convert UUID to string and generate addr string */ - uuid_unparse(hg_core_addr->na_sm_uuid, uuid_str); + /* Convert host ID to string and generate addr string */ + na_ret = NA_SM_Host_id_to_string(hg_core_addr->host_id, uuid_str); + HG_CHECK_ERROR(na_ret != NA_SUCCESS, done, ret, (hg_return_t) na_ret, + "NA_SM_Host_id_to_string() failed (%s)", + NA_Error_to_string(na_ret)); + desc_len = snprintf(addr_str, HG_CORE_ADDR_MAX_SIZE, "uid://%s" HG_CORE_ADDR_DELIMITER, uuid_str); HG_CHECK_ERROR(desc_len > HG_CORE_ADDR_MAX_SIZE, done, ret, diff --git a/src/na/CMakeLists.txt b/src/na/CMakeLists.txt index b6ca33af..930f6c4b 100644 --- a/src/na/CMakeLists.txt +++ b/src/na/CMakeLists.txt @@ -65,21 +65,17 @@ include(CheckIncludeFiles) option(NA_USE_BMI "Use BMI." OFF) if(NA_USE_BMI) find_package(BMI REQUIRED) - if(BMI_FOUND) - message(STATUS "BMI include directory: ${BMI_INCLUDE_DIR}") - set(NA_PLUGINS ${NA_PLUGINS} bmi) - set(NA_HAS_BMI 1) - set(NA_INT_INCLUDE_DEPENDENCIES - ${NA_INT_INCLUDE_DEPENDENCIES} - ${BMI_INCLUDE_DIR} - ) - set(NA_EXT_LIB_DEPENDENCIES - ${NA_EXT_LIB_DEPENDENCIES} - ${BMI_LIBRARIES} - ) - else() - message(FATAL_ERROR "Could not find BMI.") - endif() + message(STATUS "BMI include directory: ${BMI_INCLUDE_DIR}") + set(NA_PLUGINS ${NA_PLUGINS} bmi) + set(NA_HAS_BMI 1) + set(NA_INT_INCLUDE_DEPENDENCIES + ${NA_INT_INCLUDE_DEPENDENCIES} + ${BMI_INCLUDE_DIR} + ) + set(NA_EXT_LIB_DEPENDENCIES + ${NA_EXT_LIB_DEPENDENCIES} + ${BMI_LIBRARIES} + ) endif() @@ -87,41 +83,32 @@ endif() option(NA_USE_MPI "Use MPI." OFF) if(NA_USE_MPI) find_package(MPI REQUIRED) - if(MPI_FOUND) - message(STATUS "MPI include directory: ${MPI_INCLUDE_PATH}") - set(NA_HAS_MPI 1) - set(NA_PLUGINS ${NA_PLUGINS} mpi) - set(NA_EXT_INCLUDE_DEPENDENCIES - ${NA_EXT_INCLUDE_DEPENDENCIES} - ${MPI_INCLUDE_PATH} + message(STATUS "MPI include directory: ${MPI_INCLUDE_PATH}") + set(NA_HAS_MPI 1) + set(NA_PLUGINS ${NA_PLUGINS} mpi) + set(NA_EXT_INCLUDE_DEPENDENCIES + ${NA_EXT_INCLUDE_DEPENDENCIES} + ${MPI_INCLUDE_PATH} + ) + set(NA_EXT_LIB_DEPENDENCIES + ${NA_EXT_LIB_DEPENDENCIES} + ${MPI_LIBRARIES} + ) + # Extra job setup for Cray MPI without ALPS support + option(NA_MPI_USE_GNI_SETUP + "Define NA_MPI_Gni_job_setup() to setup the Aries NIC resources for the job." OFF) + mark_as_advanced(NA_MPI_USE_GNI_SETUP) + if(NA_MPI_USE_GNI_SETUP) + find_package(GNI REQUIRED) + set(NA_MPI_HAS_GNI_SETUP 1) + set(NA_INT_INCLUDE_DEPENDENCIES + ${NA_INT_INCLUDE_DEPENDENCIES} + ${GNI_INCLUDE_DIRS} ) set(NA_EXT_LIB_DEPENDENCIES ${NA_EXT_LIB_DEPENDENCIES} - ${MPI_LIBRARIES} + ${GNI_LIBRARIES} ) - # Extra job setup for Cray MPI without ALPS support - option(NA_MPI_USE_GNI_SETUP - "Define NA_MPI_Gni_job_setup() to setup the Aries NIC resources for the job." OFF) - mark_as_advanced(NA_MPI_USE_GNI_SETUP) - if(NA_MPI_USE_GNI_SETUP) - find_package(GNI REQUIRED) - if(GNI_FOUND) - set(NA_MPI_HAS_GNI_SETUP 1) - set(NA_INT_INCLUDE_DEPENDENCIES - ${NA_INT_INCLUDE_DEPENDENCIES} - ${GNI_INCLUDE_DIRS} - ) - set(NA_EXT_LIB_DEPENDENCIES - ${NA_EXT_LIB_DEPENDENCIES} - ${GNI_LIBRARIES} - ) - else() - message(FATAL_ERROR "Could not find GNI.") - endif() - endif() - # we may want to add an option for MPI_VERSION >= 3 - else() - message(FATAL_ERROR "Could not find MPI.") endif() endif() @@ -129,52 +116,44 @@ endif() option(NA_USE_CCI "Use CCI." OFF) if(NA_USE_CCI) find_package(CCI REQUIRED) - if(CCI_FOUND) - message(STATUS "CCI include directory: ${CCI_INCLUDE_DIR}") - set(NA_PLUGINS ${NA_PLUGINS} cci) - set(NA_HAS_CCI 1) - set(NA_INT_INCLUDE_DEPENDENCIES - ${NA_INT_INCLUDE_DEPENDENCIES} - ${CCI_INCLUDE_DIR} - ) - set(NA_EXT_LIB_DEPENDENCIES - ${NA_EXT_LIB_DEPENDENCIES} - ${CCI_LIBRARIES} - ) - else() - message(FATAL_ERROR "Could not find CCI.") - endif() + message(STATUS "CCI include directory: ${CCI_INCLUDE_DIR}") + set(NA_PLUGINS ${NA_PLUGINS} cci) + set(NA_HAS_CCI 1) + set(NA_INT_INCLUDE_DEPENDENCIES + ${NA_INT_INCLUDE_DEPENDENCIES} + ${CCI_INCLUDE_DIR} + ) + set(NA_EXT_LIB_DEPENDENCIES + ${NA_EXT_LIB_DEPENDENCIES} + ${CCI_LIBRARIES} + ) endif() # OFI option(NA_USE_OFI "Use libfabric plugin." OFF) if(NA_USE_OFI) find_package(OFI 1.5 REQUIRED) - if(OFI_FOUND) - message(STATUS "OFI include directory: ${OFI_INCLUDE_DIR}") - set(NA_PLUGINS ${NA_PLUGINS} ofi) - set(NA_HAS_OFI 1) - # Detect - set(CMAKE_REQUIRED_INCLUDES ${OFI_INCLUDE_DIR}) - check_include_files("rdma/fi_ext_gni.h" NA_OFI_HAS_EXT_GNI_H) - if(NA_OFI_HAS_EXT_GNI_H) - option(NA_OFI_GNI_USE_UDREG "Force gni provider to use udreg instead of internal MR cache." OFF) - if(NA_OFI_GNI_USE_UDREG) - set(NA_OFI_GNI_HAS_UDREG 1) - endif() - mark_as_advanced(NA_OFI_GNI_USE_UDREG) + message(STATUS "OFI include directory: ${OFI_INCLUDE_DIR}") + set(NA_PLUGINS ${NA_PLUGINS} ofi) + set(NA_HAS_OFI 1) + # Detect + set(CMAKE_REQUIRED_INCLUDES ${OFI_INCLUDE_DIR}) + check_include_files("rdma/fi_ext_gni.h" NA_OFI_HAS_EXT_GNI_H) + if(NA_OFI_HAS_EXT_GNI_H) + option(NA_OFI_GNI_USE_UDREG "Force gni provider to use udreg instead of internal MR cache." OFF) + if(NA_OFI_GNI_USE_UDREG) + set(NA_OFI_GNI_HAS_UDREG 1) endif() - set(NA_INT_INCLUDE_DEPENDENCIES - ${NA_INT_INCLUDE_DEPENDENCIES} - ${OFI_INCLUDE_DIR} - ) - set(NA_EXT_LIB_DEPENDENCIES - ${NA_EXT_LIB_DEPENDENCIES} - ${OFI_LIBRARIES} - ) - else() - message(FATAL_ERROR "Could not find OFI.") + mark_as_advanced(NA_OFI_GNI_USE_UDREG) endif() + set(NA_INT_INCLUDE_DEPENDENCIES + ${NA_INT_INCLUDE_DEPENDENCIES} + ${OFI_INCLUDE_DIR} + ) + set(NA_EXT_LIB_DEPENDENCIES + ${NA_EXT_LIB_DEPENDENCIES} + ${OFI_LIBRARIES} + ) endif() # SM @@ -183,6 +162,15 @@ if(NA_USE_SM) if(WIN32) message(WARNING "SM plugin not supported on this platform yet.") else() + find_package(UUID REQUIRED) + set(NA_INT_INCLUDE_DEPENDENCIES + ${NA_INT_INCLUDE_DEPENDENCIES} + ${UUID_INCLUDE_DIRS} + ) + set(NA_EXT_LIB_DEPENDENCIES + ${NA_EXT_LIB_DEPENDENCIES} + ${UUID_LIBRARIES} + ) include(CheckFunctionExists) check_function_exists(process_vm_readv NA_SM_HAS_CMA) if(NA_SM_HAS_CMA) @@ -343,6 +331,13 @@ if(NA_HAS_MPI) ) endif() +if(NA_HAS_SM) + set(NA_HEADERS + ${NA_HEADERS} + ${CMAKE_CURRENT_SOURCE_DIR}/na_sm.h + ) +endif() + #----------------------------------------------------------------------------- # Add file(s) to CMake Install #----------------------------------------------------------------------------- diff --git a/src/na/na_sm.c b/src/na/na_sm.c index 7933be68..a3737fbb 100644 --- a/src/na/na_sm.c +++ b/src/na/na_sm.c @@ -11,6 +11,7 @@ #if !defined(_WIN32) && !defined(_GNU_SOURCE) # define _GNU_SOURCE #endif +#include "na_sm.h" #include "na_plugin.h" #include "mercury_event.h" @@ -28,6 +29,8 @@ #include #include +#include + #ifdef _WIN32 # include #else @@ -946,6 +949,75 @@ lltoa(hg_util_uint64_t val, char *string, int radix) } #endif +/*---------------------------------------------------------------------------*/ +na_return_t +NA_SM_Host_id_get(na_sm_id_t id) +{ + char uuid_str[NA_SM_HOST_ID_LEN + 1]; + FILE *uuid_config = NULL; + uuid_t new_uuid; + char pathname[NA_SM_MAX_FILENAME] = {'\0'}; + char *username = getlogin_safe(); + na_return_t ret = NA_SUCCESS; + int rc; + + rc = snprintf(pathname, NA_SM_MAX_FILENAME, "%s/%s_%s/uuid.cfg", + NA_SM_TMP_DIRECTORY, NA_SM_SHM_PREFIX, username); + NA_CHECK_ERROR(rc < 0 || rc > NA_SM_MAX_FILENAME, done, ret, NA_OVERFLOW, + "snprintf() failed, rc: %d", rc); + + uuid_config = fopen(pathname, "r"); + if (!uuid_config) { + /* Generate a new one */ + uuid_generate(new_uuid); + + uuid_config = fopen(pathname, "w"); + NA_CHECK_ERROR(uuid_config == NULL, done, ret, na_sm_errno_to_na(errno), + "Could not open %s for write (%s)", pathname, strerror(errno)); + uuid_unparse(new_uuid, uuid_str); + fprintf(uuid_config, "%s\n", uuid_str); + } else { + /* Get the existing one */ + fgets(uuid_str, NA_SM_HOST_ID_LEN + 1, uuid_config); + uuid_parse(uuid_str, new_uuid); + } + fclose(uuid_config); + uuid_copy(id, new_uuid); + +done: + return ret; +} + +/*---------------------------------------------------------------------------*/ +na_return_t +NA_SM_Host_id_to_string(na_sm_id_t id, char *string) +{ + uuid_unparse(id, string); + + return NA_SUCCESS; +} + +/*---------------------------------------------------------------------------*/ +na_return_t +NA_SM_String_to_host_id(const char *string, na_sm_id_t id) +{ + return (uuid_parse(string, id) == 0) ? NA_SUCCESS : NA_PROTOCOL_ERROR; +} + +/*---------------------------------------------------------------------------*/ +void +NA_SM_Host_id_copy(na_sm_id_t dst, na_sm_id_t src) +{ + uuid_copy(dst, src); +} + +/*---------------------------------------------------------------------------*/ +na_bool_t +NA_SM_Host_id_cmp(na_sm_id_t id1, na_sm_id_t id2) +{ + return (uuid_compare(id1, id2) == 0) ? NA_TRUE : NA_FALSE; +} + /*---------------------------------------------------------------------------*/ static char * getlogin_safe(void) @@ -3170,21 +3242,26 @@ na_sm_cleanup(void) { char pathname[NA_SM_MAX_FILENAME] = {'\0'}; char *username = getlogin_safe(); - int ret; + int rc; - sprintf( - pathname, "%s/%s_%s", NA_SM_TMP_DIRECTORY, NA_SM_SHM_PREFIX, username); + rc = snprintf(pathname, NA_SM_MAX_FILENAME, "%s/%s_%s", NA_SM_TMP_DIRECTORY, + NA_SM_SHM_PREFIX, username); + NA_CHECK_ERROR_NORET(rc < 0 || rc > NA_SM_MAX_FILENAME, done, + "snprintf() failed, rc: %d", rc); /* We need to remove all files first before being able to remove the * directories */ - ret = nftw(pathname, na_sm_sock_path_cleanup, NA_SM_CLEANUP_NFDS, + rc = nftw(pathname, na_sm_sock_path_cleanup, NA_SM_CLEANUP_NFDS, FTW_PHYS | FTW_DEPTH); NA_CHECK_WARNING( - ret != 0 && errno != ENOENT, "nftw() failed (%s)", strerror(errno)); + rc != 0 && errno != ENOENT, "nftw() failed (%s)", strerror(errno)); - ret = nftw(NA_SM_SHM_PATH, na_sm_shm_cleanup, NA_SM_CLEANUP_NFDS, FTW_PHYS); + rc = nftw(NA_SM_SHM_PATH, na_sm_shm_cleanup, NA_SM_CLEANUP_NFDS, FTW_PHYS); NA_CHECK_WARNING( - ret != 0 && errno != ENOENT, "nftw() failed (%s)", strerror(errno)); + rc != 0 && errno != ENOENT, "nftw() failed (%s)", strerror(errno)); + +done: + return; } /*---------------------------------------------------------------------------*/ @@ -3368,8 +3445,12 @@ na_sm_addr_to_string(na_class_t NA_UNUSED *na_class, char *buf, na_size_t string_len; char addr_string[NA_SM_MAX_FILENAME] = {'\0'}; na_return_t ret = NA_SUCCESS; + int rc; - sprintf(addr_string, "sm://%d/%" SCNu8, na_sm_addr->pid, na_sm_addr->id); + rc = snprintf(addr_string, NA_SM_MAX_FILENAME, "sm://%d/%" SCNu8, + na_sm_addr->pid, na_sm_addr->id); + NA_CHECK_ERROR(rc < 0 || rc > NA_SM_MAX_FILENAME, done, ret, NA_OVERFLOW, + "snprintf() failed, rc: %d", rc); string_len = strlen(addr_string); if (buf) { diff --git a/src/na/na_sm.h b/src/na/na_sm.h new file mode 100644 index 00000000..b67a8958 --- /dev/null +++ b/src/na/na_sm.h @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2013-2019 Argonne National Laboratory, Department of Energy, + * UChicago Argonne, LLC and The HDF Group. + * All rights reserved. + * + * The full copyright notice, including terms governing use, modification, + * and redistribution, is contained in the COPYING file that can be + * found at the root of the source code distribution tree. + */ + +#ifndef NA_SM_H +#define NA_SM_H + +#include "na_types.h" + +/*************************************/ +/* Public Type and Struct Definition */ +/*************************************/ + +typedef unsigned char na_sm_id_t[16]; + +/*****************/ +/* Public Macros */ +/*****************/ + +/* String length of Host ID */ +#define NA_SM_HOST_ID_LEN 36 + +/*********************/ +/* Public Prototypes */ +/*********************/ + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Get the curent host ID (generate a new one if none exists). + * + * \param id [IN/OUT] SM host ID + * + * \return NA_SUCCESS or corresponding NA error code + */ +NA_PUBLIC na_return_t +NA_SM_Host_id_get(na_sm_id_t id); + +/** + * Convert host ID to string. String size must be NA_SM_HOST_ID_LEN + 1. + * + * \param id [IN] SM host ID + * \param string [IN/OUT] pointer to string + * + * \return NA_SUCCESS or corresponding NA error code + */ +NA_PUBLIC na_return_t +NA_SM_Host_id_to_string(na_sm_id_t id, char *string); + +/** + * Convert string to host ID. String size must be NA_SM_HOST_ID_LEN + 1. + * + * \param string [IN] pointer to string + * \param id [IN/OUT] SM host ID + * + * \return NA_SUCCESS or corresponding NA error code + */ +NA_PUBLIC na_return_t +NA_SM_String_to_host_id(const char *string, na_sm_id_t id); + +/** + * Copy src host ID to dst. + * + * \param dst [IN/OUT] destination SM host ID + * \param src [IN] source SM host ID + */ +NA_PUBLIC void +NA_SM_Host_id_copy(na_sm_id_t dst, na_sm_id_t src); + +/** + * Compare two host IDs. + * + * \param id1 [IN/OUT] SM host ID + * \param id2 [IN/OUT] SM host ID + * + * \return NA_TRUE if equal or NA_FALSE otherwise + */ +NA_PUBLIC na_bool_t +NA_SM_Host_id_cmp(na_sm_id_t id1, na_sm_id_t id2); + +#ifdef __cplusplus +} +#endif + +#endif /* NA_SM_H */ From 8bea46f9d7d59b62d2c5deaeb592c59df9c66a84 Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Wed, 10 Jun 2020 18:05:48 -0500 Subject: [PATCH 18/30] HG: fix addr_to_string on unexpected SM address with auto_sm (fix #324) HG Bulk: fix serialization size of handle when using auto SM --- src/mercury_bulk.c | 2 +- src/mercury_core.c | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/mercury_bulk.c b/src/mercury_bulk.c index 7223befd..7399b488 100644 --- a/src/mercury_bulk.c +++ b/src/mercury_bulk.c @@ -1408,7 +1408,7 @@ HG_Bulk_serialize(void *buf, hg_size_t buf_size, hg_bool_t request_eager, if (hg_bulk->na_sm_mem_handles) { if (hg_bulk->na_sm_mem_handles[i]) { serialize_size = NA_Mem_handle_get_serialize_size( - na_class, hg_bulk->na_sm_mem_handles[i]); + na_sm_class, hg_bulk->na_sm_mem_handles[i]); } else serialize_size = 0; ret = hg_bulk_serialize_memcpy(&buf_ptr, &buf_size_left, diff --git a/src/mercury_core.c b/src/mercury_core.c index a259fc18..db0c2355 100644 --- a/src/mercury_core.c +++ b/src/mercury_core.c @@ -1212,7 +1212,8 @@ hg_core_addr_lookup(struct hg_core_private_class *hg_core_class, /* Compare IDs, if they match it's local address */ if (hg_core_class->core_class.na_sm_class && NA_SM_Host_id_cmp( - hg_core_addr->host_id, hg_core_class->host_id) == 0) { + hg_core_addr->host_id, hg_core_class->host_id)) { + HG_LOG_DEBUG("This is a local address"); name_str = local_name; na_class = hg_core_class->core_class.na_sm_class; } else { @@ -1411,7 +1412,7 @@ hg_core_addr_to_string(struct hg_core_private_class *hg_core_class, char *buf, #endif /* Get NA address string */ - na_ret = NA_Addr_to_string(hg_core_class->core_class.na_class, buf_ptr, + na_ret = NA_Addr_to_string(hg_core_addr->core_addr.na_class, buf_ptr, &new_buf_size, hg_core_addr->core_addr.na_addr); HG_CHECK_ERROR(na_ret != NA_SUCCESS, done, ret, (hg_return_t) na_ret, "Could not convert address to string (%s)", NA_Error_to_string(na_ret)); From 3964e6de8d61dc1a1345f8b520c0cbd491b69b8d Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Thu, 11 Jun 2020 12:34:35 -0500 Subject: [PATCH 19/30] NA SM: use gethostid() by default for host IDs and use UUIDs as an option --- src/mercury_core.c | 6 +++--- src/na/CMakeLists.txt | 23 ++++++++++++-------- src/na/na_config.h.in | 1 + src/na/na_sm.c | 50 +++++++++++++++++++++++++++++++++++++------ src/na/na_sm.h | 26 ++++++++++++++-------- 5 files changed, 78 insertions(+), 28 deletions(-) diff --git a/src/mercury_core.c b/src/mercury_core.c index db0c2355..a1030823 100644 --- a/src/mercury_core.c +++ b/src/mercury_core.c @@ -1029,7 +1029,7 @@ hg_core_init(const char *na_info_string, hg_bool_t na_listen, ret, HG_NA_ERROR, "Could not initialize NA SM class"); /* Get SM host ID */ - na_ret = NA_SM_Host_id_get(hg_core_class->host_id); + na_ret = NA_SM_Host_id_get(&hg_core_class->host_id); HG_CHECK_ERROR(na_ret != NA_SUCCESS, error, ret, (hg_return_t) na_ret, "NA_SM_Host_id_get() failed (%s)", NA_Error_to_string(na_ret)); } @@ -1201,7 +1201,7 @@ hg_core_addr_lookup(struct hg_core_private_class *hg_core_class, /* Get address SM host ID */ strtok_r(lookup_name, HG_CORE_PROTO_DELIMITER, &local_id_str); na_ret = NA_SM_String_to_host_id( - local_id_str + 2, hg_core_addr->host_id); + local_id_str + 2, &hg_core_addr->host_id); HG_CHECK_ERROR(na_ret != NA_SUCCESS, error, ret, (hg_return_t) na_ret, "NA_SM_String_to_host_id() failed (%s)", NA_Error_to_string(na_ret)); @@ -1310,7 +1310,7 @@ hg_core_addr_self(struct hg_core_private_class *hg_core_class, "Could not get self SM address (%s)", NA_Error_to_string(na_ret)); /* Copy local host ID */ - NA_SM_Host_id_copy(hg_core_addr->host_id, hg_core_class->host_id); + NA_SM_Host_id_copy(&hg_core_addr->host_id, hg_core_class->host_id); } #endif diff --git a/src/na/CMakeLists.txt b/src/na/CMakeLists.txt index 930f6c4b..a3920b2e 100644 --- a/src/na/CMakeLists.txt +++ b/src/na/CMakeLists.txt @@ -162,15 +162,20 @@ if(NA_USE_SM) if(WIN32) message(WARNING "SM plugin not supported on this platform yet.") else() - find_package(UUID REQUIRED) - set(NA_INT_INCLUDE_DEPENDENCIES - ${NA_INT_INCLUDE_DEPENDENCIES} - ${UUID_INCLUDE_DIRS} - ) - set(NA_EXT_LIB_DEPENDENCIES - ${NA_EXT_LIB_DEPENDENCIES} - ${UUID_LIBRARIES} - ) + option(NA_SM_USE_UUID "Use UUIDs for host identification instead of standard host ID." OFF) + if(NA_SM_USE_UUID) + find_package(UUID REQUIRED) + set(NA_SM_HAS_UUID 1) + set(NA_INT_INCLUDE_DEPENDENCIES + ${NA_INT_INCLUDE_DEPENDENCIES} + ${UUID_INCLUDE_DIRS} + ) + set(NA_EXT_LIB_DEPENDENCIES + ${NA_EXT_LIB_DEPENDENCIES} + ${UUID_LIBRARIES} + ) + endif() + mark_as_advanced(NA_SM_USE_UUID) include(CheckFunctionExists) check_function_exists(process_vm_readv NA_SM_HAS_CMA) if(NA_SM_HAS_CMA) diff --git a/src/na/na_config.h.in b/src/na/na_config.h.in index 17a9bdfd..fcab504d 100644 --- a/src/na/na_config.h.in +++ b/src/na/na_config.h.in @@ -122,6 +122,7 @@ typedef na_uint64_t na_ptr_t; /* NA SM */ #cmakedefine NA_HAS_SM +#cmakedefine NA_SM_HAS_UUID #cmakedefine NA_SM_HAS_CMA #cmakedefine NA_SM_SHM_PREFIX "@NA_SM_SHM_PREFIX@" #cmakedefine NA_SM_TMP_DIRECTORY "@NA_SM_TMP_DIRECTORY@" diff --git a/src/na/na_sm.c b/src/na/na_sm.c index a3737fbb..5fc2b9d5 100644 --- a/src/na/na_sm.c +++ b/src/na/na_sm.c @@ -29,7 +29,9 @@ #include #include -#include +#ifdef NA_SM_HAS_UUID +# include +#endif #ifdef _WIN32 # include @@ -951,8 +953,9 @@ lltoa(hg_util_uint64_t val, char *string, int radix) /*---------------------------------------------------------------------------*/ na_return_t -NA_SM_Host_id_get(na_sm_id_t id) +NA_SM_Host_id_get(na_sm_id_t *id) { +#ifdef NA_SM_HAS_UUID char uuid_str[NA_SM_HOST_ID_LEN + 1]; FILE *uuid_config = NULL; uuid_t new_uuid; @@ -982,40 +985,73 @@ NA_SM_Host_id_get(na_sm_id_t id) uuid_parse(uuid_str, new_uuid); } fclose(uuid_config); - uuid_copy(id, new_uuid); + uuid_copy(*id, new_uuid); done: return ret; +#else + *id = gethostid(); + + return NA_SUCCESS; +#endif } /*---------------------------------------------------------------------------*/ na_return_t NA_SM_Host_id_to_string(na_sm_id_t id, char *string) { +#ifdef NA_SM_HAS_UUID uuid_unparse(id, string); return NA_SUCCESS; +#else + na_return_t ret = NA_SUCCESS; + int rc = snprintf(string, NA_SM_HOST_ID_LEN + 1, "%ld", id); + NA_CHECK_ERROR(rc < 0 || rc > NA_SM_HOST_ID_LEN + 1, done, ret, NA_OVERFLOW, + "snprintf() failed, rc: %d", rc); + +done: + return ret; +#endif } /*---------------------------------------------------------------------------*/ na_return_t -NA_SM_String_to_host_id(const char *string, na_sm_id_t id) +NA_SM_String_to_host_id(const char *string, na_sm_id_t *id) { - return (uuid_parse(string, id) == 0) ? NA_SUCCESS : NA_PROTOCOL_ERROR; +#ifdef NA_SM_HAS_UUID + return (uuid_parse(string, *id) == 0) ? NA_SUCCESS : NA_PROTOCOL_ERROR; +#else + na_return_t ret = NA_SUCCESS; + int rc = sscanf(string, "%ld", id); + NA_CHECK_ERROR(rc != 1, done, ret, NA_PROTOCOL_ERROR, + "sscanf() failed, rc: %d", rc); + +done: + return ret; +#endif } /*---------------------------------------------------------------------------*/ void -NA_SM_Host_id_copy(na_sm_id_t dst, na_sm_id_t src) +NA_SM_Host_id_copy(na_sm_id_t *dst, na_sm_id_t src) { - uuid_copy(dst, src); +#ifdef NA_SM_HAS_UUID + uuid_copy(*dst, src); +#else + *dst = src; +#endif } /*---------------------------------------------------------------------------*/ na_bool_t NA_SM_Host_id_cmp(na_sm_id_t id1, na_sm_id_t id2) { +#ifdef NA_SM_HAS_UUID return (uuid_compare(id1, id2) == 0) ? NA_TRUE : NA_FALSE; +#else + return (id1 == id2); +#endif } /*---------------------------------------------------------------------------*/ diff --git a/src/na/na_sm.h b/src/na/na_sm.h index b67a8958..a119d5b1 100644 --- a/src/na/na_sm.h +++ b/src/na/na_sm.h @@ -17,14 +17,22 @@ /* Public Type and Struct Definition */ /*************************************/ +#ifdef NA_SM_HAS_UUID typedef unsigned char na_sm_id_t[16]; +#else +typedef long na_sm_id_t; +#endif /*****************/ /* Public Macros */ /*****************/ /* String length of Host ID */ -#define NA_SM_HOST_ID_LEN 36 +#ifdef NA_SM_HAS_UUID +# define NA_SM_HOST_ID_LEN 36 +#else +# define NA_SM_HOST_ID_LEN 11 +#endif /*********************/ /* Public Prototypes */ @@ -37,12 +45,12 @@ extern "C" { /** * Get the curent host ID (generate a new one if none exists). * - * \param id [IN/OUT] SM host ID + * \param id [IN/OUT] pointer to SM host ID * * \return NA_SUCCESS or corresponding NA error code */ NA_PUBLIC na_return_t -NA_SM_Host_id_get(na_sm_id_t id); +NA_SM_Host_id_get(na_sm_id_t *id); /** * Convert host ID to string. String size must be NA_SM_HOST_ID_LEN + 1. @@ -59,27 +67,27 @@ NA_SM_Host_id_to_string(na_sm_id_t id, char *string); * Convert string to host ID. String size must be NA_SM_HOST_ID_LEN + 1. * * \param string [IN] pointer to string - * \param id [IN/OUT] SM host ID + * \param id [IN/OUT] pointer to SM host ID * * \return NA_SUCCESS or corresponding NA error code */ NA_PUBLIC na_return_t -NA_SM_String_to_host_id(const char *string, na_sm_id_t id); +NA_SM_String_to_host_id(const char *string, na_sm_id_t *id); /** * Copy src host ID to dst. * - * \param dst [IN/OUT] destination SM host ID + * \param dst [IN/OUT] pointer to destination SM host ID * \param src [IN] source SM host ID */ NA_PUBLIC void -NA_SM_Host_id_copy(na_sm_id_t dst, na_sm_id_t src); +NA_SM_Host_id_copy(na_sm_id_t *dst, na_sm_id_t src); /** * Compare two host IDs. * - * \param id1 [IN/OUT] SM host ID - * \param id2 [IN/OUT] SM host ID + * \param id1 [IN] SM host ID + * \param id2 [IN] SM host ID * * \return NA_TRUE if equal or NA_FALSE otherwise */ From f41bec641897767dc19acb8f13ac44b5ea799725 Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Thu, 11 Jun 2020 18:00:08 -0500 Subject: [PATCH 20/30] NA Test: fix test_lat_client test --- Testing/na/CMakeLists.txt | 2 +- Testing/na/test_lat_client.c | 42 +++--------------------------------- 2 files changed, 4 insertions(+), 40 deletions(-) diff --git a/Testing/na/CMakeLists.txt b/Testing/na/CMakeLists.txt index 04466fce..6a78b3b3 100644 --- a/Testing/na/CMakeLists.txt +++ b/Testing/na/CMakeLists.txt @@ -75,7 +75,7 @@ endif() build_na_test(server) #build_na_test(cancel_client) build_na_test(cancel_server) -#build_na_test(lat_client) +build_na_test(lat_client) build_na_test(lat_server) #------------------------------------------------------------------------------ diff --git a/Testing/na/test_lat_client.c b/Testing/na/test_lat_client.c index cd03c5ea..9cc49f95 100644 --- a/Testing/na/test_lat_client.c +++ b/Testing/na/test_lat_client.c @@ -47,11 +47,6 @@ struct na_test_lat_info { struct na_test_info na_test_info; }; -struct na_test_target_lookup_arg { - na_addr_t *addr_ptr; - hg_request_t *request; -}; - /********************/ /* Local Prototypes */ /********************/ @@ -65,9 +60,6 @@ na_test_request_trigger(unsigned int timeout, unsigned int *flag, void *arg); static na_return_t na_test_target_lookup(struct na_test_lat_info *na_test_lat_info); -static NA_INLINE int -na_test_target_lookup_cb(const struct na_cb_info *na_cb_info); - static NA_INLINE int na_test_recv_expected_cb(const struct na_cb_info *na_cb_info); @@ -118,49 +110,21 @@ na_test_request_trigger(unsigned int timeout, unsigned int *flag, void *arg) static na_return_t na_test_target_lookup(struct na_test_lat_info *na_test_lat_info) { - struct na_test_target_lookup_arg request_args = { 0 }; - hg_request_t *request = NULL; - na_op_id_t op_id = NA_OP_ID_NULL; na_return_t ret = NA_SUCCESS; - request = hg_request_create(na_test_lat_info->request_class); - request_args.addr_ptr = &na_test_lat_info->target_addr; - request_args.request = request; - - op_id = NA_Op_create(na_test_lat_info->na_class); - /* Forward call to remote addr and get a new request */ - ret = NA_Addr_lookup(na_test_lat_info->na_class, na_test_lat_info->context, - na_test_target_lookup_cb, &request_args, - na_test_lat_info->na_test_info.target_name, &op_id); + ret = NA_Addr_lookup(na_test_lat_info->na_class, + na_test_lat_info->na_test_info.target_name, + &na_test_lat_info->target_addr); if (ret != NA_SUCCESS) { NA_LOG_ERROR("Could not lookup address (%s)", NA_Error_to_string(ret)); goto done; } - /* Wait for request to be marked completed */ - hg_request_wait(request, NA_MAX_IDLE_TIME, NULL); - done: - NA_Op_destroy(na_test_lat_info->na_class, op_id); - hg_request_destroy(request); return ret; } -/*---------------------------------------------------------------------------*/ -static NA_INLINE int -na_test_target_lookup_cb(const struct na_cb_info *na_cb_info) -{ - struct na_test_target_lookup_arg *na_test_target_lookup_arg = - (struct na_test_target_lookup_arg *) na_cb_info->arg; - - *na_test_target_lookup_arg->addr_ptr = na_cb_info->info.lookup.addr; - - hg_request_complete(na_test_target_lookup_arg->request); - - return NA_SUCCESS; -} - /*---------------------------------------------------------------------------*/ static NA_INLINE int na_test_recv_expected_cb(const struct na_cb_info *na_cb_info) From 746123d82104b3205ea5f36445d184275082aa05 Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Thu, 18 Jun 2020 13:27:14 -0500 Subject: [PATCH 21/30] HG util: Rework mercury poll and no longer use callbacks --- Testing/util/test_poll.c | 116 ++++--- src/util/mercury_poll.c | 630 ++++++++++++++++++--------------------- src/util/mercury_poll.h | 79 ++--- 3 files changed, 396 insertions(+), 429 deletions(-) diff --git a/Testing/util/test_poll.c b/Testing/util/test_poll.c index 878cbc5c..8f55f3c1 100644 --- a/Testing/util/test_poll.c +++ b/Testing/util/test_poll.c @@ -1,102 +1,144 @@ -#include "mercury_poll.h" #include "mercury_event.h" +#include "mercury_poll.h" #include "mercury_test_config.h" #include #include -struct hg_test_poll_cb_args { - int event_fd; -}; - -static int -poll_cb(void *arg, int error, struct hg_poll_event *event) -{ - struct hg_test_poll_cb_args *poll_cb_args = - (struct hg_test_poll_cb_args *) arg; - (void) error; - - hg_event_get(poll_cb_args->event_fd, &event->progressed); - - return HG_UTIL_SUCCESS; -} - int main(void) { - struct hg_test_poll_cb_args poll_cb_args; hg_poll_set_t *poll_set; - struct hg_poll_event events[1]; + struct hg_poll_event events[2]; unsigned int nevents = 0; - int event_fd, ret = EXIT_SUCCESS; + hg_util_bool_t signaled = HG_UTIL_FALSE; + int event_fd1, event_fd2, ret = EXIT_SUCCESS; poll_set = hg_poll_create(); - event_fd = hg_event_create(); - - poll_cb_args.event_fd = event_fd; + event_fd1 = hg_event_create(); + event_fd2 = hg_event_create(); /* Add event descriptor */ - hg_poll_add(poll_set, event_fd, HG_POLLIN, poll_cb, &poll_cb_args); + events[0].events = HG_POLLIN; + events[1].events = HG_POLLIN; + + hg_poll_add(poll_set, event_fd1, &events[0]); + hg_poll_add(poll_set, event_fd2, &events[1]); /* Set event */ - hg_event_set(event_fd); + hg_event_set(event_fd1); /* Wait with timeout 0 */ hg_poll_wait(poll_set, 0, 1, events, &nevents); - if ((nevents != 1) || !events[0].progressed) { + if (nevents != 1) { /* We expect success */ fprintf(stderr, "Error: should have progressed\n"); ret = EXIT_FAILURE; goto done; } + hg_event_get(event_fd1, &signaled); + if (!signaled) { + /* We expect success */ + fprintf(stderr, "Error: should have been signaled\n"); + ret = EXIT_FAILURE; + goto done; + } /* Reset progressed */ nevents = 0; - events[0].progressed = HG_UTIL_FALSE; /* Wait with timeout 0 */ hg_poll_wait(poll_set, 0, 1, events, &nevents); - if ((nevents != 1) || events[0].progressed) { + if (nevents) { /* We do not expect success */ - fprintf(stderr, "Error: should not have progressed\n"); + fprintf(stderr, "Error: should not have progressed (timeout 0)\n"); ret = EXIT_FAILURE; goto done; } /* Reset progressed */ nevents = 0; - events[0].progressed = HG_UTIL_FALSE; /* Wait with timeout */ hg_poll_wait(poll_set, 100, 1, events, &nevents); - if (nevents || events[0].progressed) { + if (nevents) { /* We do not expect success */ - fprintf(stderr, "Error: should not have progressed\n"); + fprintf(stderr, "Error: should not have progressed (timeout 100)\n"); ret = EXIT_FAILURE; goto done; } /* Set event */ - hg_event_set(event_fd); + hg_event_set(event_fd1); /* Reset progressed */ nevents = 0; - events[0].progressed = HG_UTIL_FALSE; /* Wait with timeout */ hg_poll_wait(poll_set, 1000, 1, events, &nevents); - if (!nevents || !events[0].progressed) { + if (!nevents) { /* We expect success */ fprintf(stderr, "Error: did not progress correctly\n"); ret = EXIT_FAILURE; goto done; } + hg_event_get(event_fd1, &signaled); + if (!signaled) { + /* We expect success */ + fprintf(stderr, "Error: should have been signaled\n"); + ret = EXIT_FAILURE; + goto done; + } + + /* Set event */ + hg_event_set(event_fd1); + hg_event_set(event_fd2); + + /* Reset progressed */ + nevents = 0; + + /* Wait with timeout */ + hg_poll_wait(poll_set, 1000, 1, events, &nevents); + if (nevents != 1) { + /* We do not expect success */ + fprintf(stderr, "Error: should not have progressed first time\n"); + ret = EXIT_FAILURE; + goto done; + } + hg_event_get(event_fd1, &signaled); + if (!signaled) { + /* We expect success */ + fprintf(stderr, "Error: should have been signaled\n"); + ret = EXIT_FAILURE; + goto done; + } + + /* Reset progressed */ + nevents = 0; + + /* Wait with timeout */ + hg_poll_wait(poll_set, 1000, 2, events, &nevents); + if (nevents != 1) { + /* We expect success */ + fprintf(stderr, "Error: did not progress second time\n"); + ret = EXIT_FAILURE; + goto done; + } + hg_event_get(event_fd2, &signaled); + if (!signaled) { + /* We expect success */ + fprintf(stderr, "Error: should have been signaled\n"); + ret = EXIT_FAILURE; + goto done; + } done: - hg_poll_remove(poll_set, event_fd); + hg_poll_remove(poll_set, event_fd1); + hg_poll_remove(poll_set, event_fd2); hg_poll_destroy(poll_set); - hg_event_destroy(event_fd); + hg_event_destroy(event_fd1); + hg_event_destroy(event_fd2); return ret; } diff --git a/src/util/mercury_poll.c b/src/util/mercury_poll.c index 57f7e3e7..789657ab 100644 --- a/src/util/mercury_poll.c +++ b/src/util/mercury_poll.c @@ -9,10 +9,8 @@ */ #include "mercury_poll.h" -#include "mercury_atomic.h" #include "mercury_event.h" -#include "mercury_list.h" -#include "mercury_thread_spin.h" +#include "mercury_thread_mutex.h" #include "mercury_util_error.h" #include @@ -37,7 +35,8 @@ /* Local Macros */ /****************/ -#define HG_POLL_MAX_EVENTS 1024 +#define HG_POLL_INIT_NEVENTS 32 +#define HG_POLL_MAX_EVENTS 4096 #ifndef MIN # define MIN(a, b) (((a) < (b)) ? (a) : (b)) @@ -47,26 +46,19 @@ /* Local Type and Struct Definition */ /************************************/ -struct hg_poll_data { +struct hg_poll_set { + hg_thread_mutex_t lock; #if defined(HG_UTIL_HAS_SYSEPOLL_H) - int fd; + struct epoll_event *events; #elif defined(HG_UTIL_HAS_SYSEVENT_H) - struct kevent kev; + struct kevent *events; #else - struct pollfd pollfd; + struct pollfd *events; + hg_poll_data_t *event_data; #endif - hg_poll_cb_t poll_cb; - void *poll_arg; - HG_LIST_ENTRY(hg_poll_data) entry; -}; - -struct hg_poll_set { + unsigned int max_events; + unsigned int nfds; int fd; - hg_atomic_int32_t nfds; - hg_poll_try_wait_cb_t try_wait_cb; - void *try_wait_arg; - HG_LIST_HEAD(hg_poll_data) poll_data_list; - hg_thread_spin_t poll_data_list_lock; }; /********************/ @@ -85,35 +77,46 @@ hg_poll_create(void) hg_poll_set = malloc(sizeof(struct hg_poll_set)); HG_UTIL_CHECK_ERROR_NORET( - hg_poll_set == NULL, error, "malloc() failed (%s)"); + hg_poll_set == NULL, error, "malloc() failed (%s)", strerror(errno)); + + hg_thread_mutex_init(&hg_poll_set->lock); + hg_poll_set->nfds = 0; + hg_poll_set->max_events = HG_POLL_INIT_NEVENTS; + + /* Preallocate events, size will grow as needed */ + hg_poll_set->events = + malloc(sizeof(*hg_poll_set->events) * hg_poll_set->max_events); + HG_UTIL_CHECK_ERROR_NORET( + !hg_poll_set->events, error, "malloc() failed (%s)", strerror(errno)); + #if defined(_WIN32) /* TODO */ -#else - HG_LIST_INIT(&hg_poll_set->poll_data_list); - hg_thread_spin_init(&hg_poll_set->poll_data_list_lock); - hg_atomic_init32(&hg_poll_set->nfds, 0); - hg_poll_set->try_wait_cb = NULL; - -# if defined(HG_UTIL_HAS_SYSEPOLL_H) +#elif defined(HG_UTIL_HAS_SYSEPOLL_H) hg_poll_set->fd = epoll_create1(0); HG_UTIL_CHECK_ERROR_NORET(hg_poll_set->fd == -1, error, "epoll_create1() failed (%s)", strerror(errno)); -# elif defined(HG_UTIL_HAS_SYSEVENT_H) +#elif defined(HG_UTIL_HAS_SYSEVENT_H) hg_poll_set->fd = kqueue(); HG_UTIL_CHECK_ERROR_NORET( hg_poll_set->fd == -1, error, "kqueue() failed (%s)", strerror(errno)); -# else +#else hg_poll_set->fd = hg_event_create(); HG_UTIL_CHECK_ERROR_NORET(hg_poll_set->fd == -1, error, "hg_event_create() failed (%s)", strerror(errno)); -# endif -#endif /* defined(_WIN32) */ + + /* Preallocate event_data, size will grow as needed */ + hg_poll_set->event_data = + malloc(sizeof(*hg_poll_set->event_data) * hg_poll_set->max_events); + HG_UTIL_CHECK_ERROR_NORET( + !hg_poll_set->events, error, "malloc() failed (%s)", strerror(errno)); +#endif return hg_poll_set; error: if (hg_poll_set) { - hg_thread_spin_destroy(&hg_poll_set->poll_data_list_lock); + free(hg_poll_set->events); + hg_thread_mutex_destroy(&hg_poll_set->lock); free(hg_poll_set); } return NULL; @@ -129,26 +132,28 @@ hg_poll_destroy(hg_poll_set_t *poll_set) if (!poll_set) goto done; -#if defined(_WIN32) - /* TODO */ -#else - HG_UTIL_CHECK_ERROR(hg_atomic_get32(&poll_set->nfds), done, ret, - HG_UTIL_FAIL, "Poll set non empty"); + HG_UTIL_CHECK_ERROR( + poll_set->nfds > 0, done, ret, HG_UTIL_FAIL, "Poll set non empty"); -# if defined(HG_UTIL_HAS_SYSEPOLL_H) || defined(HG_UTIL_HAS_SYSEVENT_H) +#if defined(_WIN32) + /* TODO */ +#elif defined(HG_UTIL_HAS_SYSEPOLL_H) || defined(HG_UTIL_HAS_SYSEVENT_H) /* Close poll descriptor */ rc = close(poll_set->fd); HG_UTIL_CHECK_ERROR(rc == -1, done, ret, HG_UTIL_FAIL, "close() failed (%s)", strerror(errno)); -# else +#else rc = hg_event_destroy(poll_set->fd); HG_UTIL_CHECK_ERROR(rc == HG_UTIL_FAIL, done, ret, HG_UTIL_FAIL, "hg_event_destroy() failed (%s)", strerror(errno)); -# endif - - hg_thread_spin_destroy(&poll_set->poll_data_list_lock); -#endif /* defined(_WIN32) */ +#endif + hg_thread_mutex_destroy(&poll_set->lock); +#if !defined(_WIN32) && !defined(HG_UTIL_HAS_SYSEPOLL_H) && \ + !defined(HG_UTIL_HAS_SYSEVENT_H) + free(poll_set->event_data); +#endif + free(poll_set->events); free(poll_set); done: @@ -159,208 +164,176 @@ hg_poll_destroy(hg_poll_set_t *poll_set) int hg_poll_get_fd(hg_poll_set_t *poll_set) { - int fd = -1; - - HG_UTIL_CHECK_ERROR_NORET(!poll_set, done, "NULL poll set"); - #if defined(_WIN32) /* TODO */ + return -1; #else - fd = poll_set->fd; + return poll_set->fd; #endif - -done: - return fd; } /*---------------------------------------------------------------------------*/ int -hg_poll_set_try_wait( - hg_poll_set_t *poll_set, hg_poll_try_wait_cb_t try_wait_cb, void *arg) +hg_poll_add(hg_poll_set_t *poll_set, int fd, struct hg_poll_event *event) { +#if defined(_WIN32) + /* TODO */ +#elif defined(HG_UTIL_HAS_SYSEPOLL_H) + struct epoll_event ev; + uint32_t poll_flags = 0; + int rc; +#elif defined(HG_UTIL_HAS_SYSEVENT_H) + struct kevent ev; + struct timespec timeout = {0, 0}; + int16_t poll_flags = 0; + int rc; +#else + struct pollfd ev; + short int poll_flags = 0; +#endif int ret = HG_UTIL_SUCCESS; - HG_UTIL_CHECK_ERROR(!poll_set, done, ret, HG_UTIL_FAIL, "NULL poll set"); - - poll_set->try_wait_cb = try_wait_cb; - poll_set->try_wait_arg = arg; - -done: - return ret; -} - -/*---------------------------------------------------------------------------*/ -int -hg_poll_add(hg_poll_set_t *poll_set, int fd, unsigned int flags, - hg_poll_cb_t poll_cb, void *poll_arg) -{ - struct hg_poll_data *hg_poll_data = NULL; - int ret = HG_UTIL_SUCCESS; - - HG_UTIL_CHECK_ERROR(!poll_set, done, ret, HG_UTIL_FAIL, "NULL poll set"); - - /* Allocate poll data that can hold user data and callback */ - hg_poll_data = malloc(sizeof(struct hg_poll_data)); - HG_UTIL_CHECK_ERROR( - !hg_poll_data, done, ret, HG_UTIL_FAIL, "malloc() failed (%s)"); - memset(hg_poll_data, 0, sizeof(struct hg_poll_data)); - hg_poll_data->poll_cb = poll_cb; - hg_poll_data->poll_arg = poll_arg; + HG_UTIL_CHECK_ERROR(fd <= STDERR_FILENO, done, ret, HG_UTIL_FAIL, + "fd is not valid (%d)", fd); - if (fd > 0) { #if defined(_WIN32) - /* TODO */ + /* TODO */ #elif defined(HG_UTIL_HAS_SYSEPOLL_H) - struct epoll_event ev; - uint32_t poll_flags; - int rc; - - /* Translate flags */ - switch (flags) { - case HG_POLLIN: - poll_flags = EPOLLIN; - break; - case HG_POLLOUT: - poll_flags = EPOLLOUT; - break; - default: - HG_UTIL_GOTO_ERROR(error, ret, HG_UTIL_FAIL, "Invalid flag"); - } - - hg_poll_data->fd = fd; - ev.events = poll_flags; - ev.data.ptr = hg_poll_data; - - rc = epoll_ctl(poll_set->fd, EPOLL_CTL_ADD, fd, &ev); - HG_UTIL_CHECK_ERROR(rc != 0, error, ret, HG_UTIL_FAIL, - "epoll_ctl() failed (%s)", strerror(errno)); + /* Translate flags */ + if (event->events & HG_POLLIN) + poll_flags |= EPOLLIN; + if (event->events & HG_POLLOUT) + poll_flags |= EPOLLOUT; + + ev.events = poll_flags; + ev.data.u64 = (uint64_t) event->data.u64; + + rc = epoll_ctl(poll_set->fd, EPOLL_CTL_ADD, fd, &ev); + HG_UTIL_CHECK_ERROR(rc != 0, done, ret, HG_UTIL_FAIL, + "epoll_ctl() failed (%s)", strerror(errno)); #elif defined(HG_UTIL_HAS_SYSEVENT_H) - struct timespec timeout = {0, 0}; - int16_t poll_flags; - int rc; - - /* Translate flags */ - switch (flags) { - case HG_POLLIN: - poll_flags = EVFILT_READ; - break; - case HG_POLLOUT: - poll_flags = EVFILT_WRITE; - break; - default: - HG_UTIL_GOTO_ERROR(error, ret, HG_UTIL_FAIL, "Invalid flag"); - } + /* Translate flags */ + if (event->events & HG_POLLIN) + poll_flags |= EVFILT_READ; + if (event->events & HG_POLLOUT) + poll_flags |= EVFILT_WRITE; - EV_SET(&hg_poll_data->kev, (uintptr_t) fd, poll_flags, EV_ADD, 0, 0, - hg_poll_data); + EV_SET(&ev, (uintptr_t) fd, poll_flags, EV_ADD, 0, 0, event->data.ptr); - rc = kevent(poll_set->fd, &hg_poll_data->kev, 1, NULL, 0, &timeout); - HG_UTIL_CHECK_ERROR(rc == -1, error, ret, HG_UTIL_FAIL, - "kevent() failed (%s)", strerror(errno)); + rc = kevent(poll_set->fd, &ev, 1, NULL, 0, &timeout); + HG_UTIL_CHECK_ERROR(rc == -1, done, ret, HG_UTIL_FAIL, + "kevent() failed (%s)", strerror(errno)); #else - short int poll_flags; - - /* Translate flags */ - switch (flags) { - case HG_POLLIN: - poll_flags = POLLIN; - break; - case HG_POLLOUT: - poll_flags = POLLOUT; - break; - default: - HG_UTIL_GOTO_ERROR(error, ret, HG_UTIL_FAIL, "Invalid flag"); - } + /* Translate flags */ + if (event->events & HG_POLLIN) + poll_flags |= POLLIN; + if (event->events & HG_POLLOUT) + poll_flags |= POLLOUT; + + ev.fd = fd; + ev.events = poll_flags; + ev.revents = 0; +#endif - hg_poll_data->pollfd.fd = fd; - hg_poll_data->pollfd.events = poll_flags; - hg_poll_data->pollfd.revents = 0; -#endif /* defined(_WIN32) */ + hg_thread_mutex_lock(&poll_set->lock); + +#if !defined(_WIN32) && !defined(HG_UTIL_HAS_SYSEPOLL_H) && \ + !defined(HG_UTIL_HAS_SYSEVENT_H) + /* Grow array if reached max number */ + if (poll_set->nfds == poll_set->max_events) { + HG_UTIL_CHECK_ERROR(poll_set->max_events * 2 > HG_POLL_MAX_EVENTS, + unlock, ret, HG_UTIL_FAIL, + "reached max number of events for this poll set (%d)", + poll_set->max_events); + + poll_set->events = realloc(poll_set->events, + sizeof(*poll_set->events) * poll_set->max_events * 2); + HG_UTIL_CHECK_ERROR(!poll_set->events, unlock, ret, HG_UTIL_FAIL, + "realloc() failed (%s)", strerror(errno)); + + poll_set->event_data = realloc(poll_set->event_data, + sizeof(*poll_set->event_data) * poll_set->max_events * 2); + HG_UTIL_CHECK_ERROR(!poll_set->event_data, unlock, ret, HG_UTIL_FAIL, + "realloc() failed (%s)", strerror(errno)); + + poll_set->max_events *= 2; } - hg_atomic_incr32(&poll_set->nfds); + poll_set->events[poll_set->nfds] = ev; + poll_set->event_data[poll_set->nfds] = event->data; +#endif + poll_set->nfds++; - hg_thread_spin_lock(&poll_set->poll_data_list_lock); - HG_LIST_INSERT_HEAD(&poll_set->poll_data_list, hg_poll_data, entry); - hg_thread_spin_unlock(&poll_set->poll_data_list_lock); +#if !defined(_WIN32) && !defined(HG_UTIL_HAS_SYSEPOLL_H) && \ + !defined(HG_UTIL_HAS_SYSEVENT_H) +unlock: +#endif + hg_thread_mutex_unlock(&poll_set->lock); done: return ret; - -error: - free(hg_poll_data); - - return HG_UTIL_FAIL; } /*---------------------------------------------------------------------------*/ int hg_poll_remove(hg_poll_set_t *poll_set, int fd) { - struct hg_poll_data *hg_poll_data; - hg_util_bool_t found = HG_UTIL_FALSE; +#if defined(_WIN32) + /* TODO */ +#elif defined(HG_UTIL_HAS_SYSEPOLL_H) + int rc; +#elif defined(HG_UTIL_HAS_SYSEVENT_H) + struct kevent ev; + struct timespec timeout = {0, 0}; + int rc; +#else + int i, found = -1; +#endif int ret = HG_UTIL_SUCCESS; - HG_UTIL_CHECK_ERROR(!poll_set, done, ret, HG_UTIL_FAIL, "NULL poll set"); + HG_UTIL_CHECK_ERROR(fd <= STDERR_FILENO, done, ret, HG_UTIL_FAIL, + "fd is not valid (%d)", fd); - hg_thread_spin_lock(&poll_set->poll_data_list_lock); - HG_LIST_FOREACH (hg_poll_data, &poll_set->poll_data_list, entry) { #if defined(_WIN32) - /* TODO */ + /* TODO */ #elif defined(HG_UTIL_HAS_SYSEPOLL_H) - if (hg_poll_data->fd == fd) { - HG_LIST_REMOVE(hg_poll_data, entry); - - if (fd > 0) { - int rc = epoll_ctl(poll_set->fd, EPOLL_CTL_DEL, fd, NULL); - HG_UTIL_CHECK_ERROR(rc != 0, error, ret, HG_UTIL_FAIL, - "epoll_ctl() failed (%s)", strerror(errno)); - } - free(hg_poll_data); - found = HG_UTIL_TRUE; - break; - } + rc = epoll_ctl(poll_set->fd, EPOLL_CTL_DEL, fd, NULL); + HG_UTIL_CHECK_ERROR(rc != 0, done, ret, HG_UTIL_FAIL, + "epoll_ctl() failed (%s)", strerror(errno)); + hg_thread_mutex_lock(&poll_set->lock); #elif defined(HG_UTIL_HAS_SYSEVENT_H) - /* Events which are attached to file descriptors are automatically - * deleted on the last close of the descriptor. */ - if ((int) hg_poll_data->kev.ident == fd) { - HG_LIST_REMOVE(hg_poll_data, entry); - - if (fd > 0) { - struct timespec timeout = {0, 0}; - int rc; - - EV_SET(&hg_poll_data->kev, (uintptr_t) fd, EVFILT_READ, - EV_DELETE, 0, 0, NULL); - rc = kevent( - poll_set->fd, &hg_poll_data->kev, 1, NULL, 0, &timeout); - HG_UTIL_CHECK_ERROR(rc == -1, error, ret, HG_UTIL_FAIL, - "kevent() failed (%s)", strerror(errno)); - } - free(hg_poll_data); - found = HG_UTIL_TRUE; - break; - } + /* Events which are attached to file descriptors are automatically + * deleted on the last close of the descriptor. */ + EV_SET(&ev, (uintptr_t) fd, EVFILT_READ, EV_DELETE, 0, 0, NULL); + rc = kevent(poll_set->fd, &ev, 1, NULL, 0, &timeout); + HG_UTIL_CHECK_ERROR(rc == -1, done, ret, HG_UTIL_FAIL, + "kevent() failed (%s)", strerror(errno)); + hg_thread_mutex_lock(&poll_set->lock); #else - if (hg_poll_data->pollfd.fd == fd) { - HG_LIST_REMOVE(hg_poll_data, entry); - free(hg_poll_data); - found = HG_UTIL_TRUE; + hg_thread_mutex_lock(&poll_set->lock); + for (i = 0; i < (int) poll_set->nfds; i++) { + if (poll_set->events[i].fd == fd) { + found = i; break; } -#endif } - hg_thread_spin_unlock(&poll_set->poll_data_list_lock); - HG_UTIL_CHECK_ERROR( - !found, done, ret, HG_UTIL_FAIL, "Could not find fd in poll_set"); - hg_atomic_decr32(&poll_set->nfds); + found < 0, error, ret, HG_UTIL_FAIL, "Could not find fd in poll_set"); + + for (i = found; i < (int) poll_set->nfds - 1; i++) { + poll_set->events[i] = poll_set->events[i + 1]; + poll_set->event_data[i] = poll_set->event_data[i + 1]; + } +#endif + poll_set->nfds--; + hg_thread_mutex_unlock(&poll_set->lock); done: return ret; -#if defined(HG_UTIL_HAS_SYSEPOLL_H) || defined(HG_UTIL_HAS_SYSEVENT_H) +#if !defined(_WIN32) && !defined(HG_UTIL_HAS_SYSEPOLL_H) && \ + !defined(HG_UTIL_HAS_SYSEVENT_H) error: - hg_thread_spin_unlock(&poll_set->poll_data_list_lock); + hg_thread_mutex_unlock(&poll_set->lock); return ret; #endif @@ -372,160 +345,151 @@ hg_poll_wait(hg_poll_set_t *poll_set, unsigned int timeout, unsigned int max_events, struct hg_poll_event *events, unsigned int *actual_events) { - int max_poll_events = (int) MIN(max_events, HG_POLL_MAX_EVENTS); + int max_poll_events = (int) MIN(max_events, poll_set->max_events); int nfds = 0, i; int ret = HG_UTIL_SUCCESS; - HG_UTIL_CHECK_ERROR(!poll_set, done, ret, HG_UTIL_FAIL, "NULL poll set"); - - if (timeout && (!poll_set->try_wait_cb || - (poll_set->try_wait_cb && - poll_set->try_wait_cb(poll_set->try_wait_arg)))) { #if defined(_WIN32) #elif defined(HG_UTIL_HAS_SYSEPOLL_H) - struct epoll_event poll_events[HG_POLL_MAX_EVENTS]; - - nfds = epoll_wait( - poll_set->fd, poll_events, max_poll_events, (int) timeout); - HG_UTIL_CHECK_ERROR(nfds == -1 && errno != EINTR, done, ret, - HG_UTIL_FAIL, "epoll_wait() failed (%s)", strerror(errno)); - - for (i = 0; i < nfds; ++i) { - struct hg_poll_data *hg_poll_data = - (struct hg_poll_data *) poll_events[i].data.ptr; - int error = 0, rc; - - HG_UTIL_CHECK_ERROR(hg_poll_data == NULL, done, ret, HG_UTIL_FAIL, - "NULL poll data"); - - /* Don't change the if/else order */ - if (poll_events[i].events & EPOLLERR) - error = EPOLLERR; - else if (poll_events[i].events & EPOLLHUP) - error = EPOLLHUP; - else if (poll_events[i].events & EPOLLRDHUP) - error = EPOLLRDHUP; - - HG_UTIL_CHECK_ERROR(!(poll_events[i].events & (EPOLLIN | EPOLLOUT)), - done, ret, HG_UTIL_FAIL, "Unsupported events"); - - if (!hg_poll_data->poll_cb) - continue; - - rc = hg_poll_data->poll_cb( - hg_poll_data->poll_arg, error, &events[i]); - HG_UTIL_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, HG_UTIL_FAIL, - "poll cb failed"); - } + nfds = epoll_wait( + poll_set->fd, poll_set->events, max_poll_events, (int) timeout); + HG_UTIL_CHECK_ERROR(nfds == -1 && errno != EINTR, done, ret, HG_UTIL_FAIL, + "epoll_wait() failed (%s)", strerror(errno)); + + for (i = 0; i < nfds; ++i) { + events[i].events = 0; + events[i].data.u64 = (hg_util_uint64_t) poll_set->events[i].data.u64; + + if (poll_set->events[i].events & EPOLLIN) + events[i].events |= HG_POLLIN; + + if (poll_set->events[i].events & EPOLLOUT) + events[i].events |= HG_POLLOUT; + + /* Don't change the if/else order */ + if (poll_set->events[i].events & EPOLLERR) + events[i].events |= HG_POLLERR; + else if (poll_set->events[i].events & EPOLLHUP) + events[i].events |= HG_POLLHUP; + else if (poll_set->events[i].events & EPOLLRDHUP) + events[i].events |= HG_POLLHUP; + } + + /* Grow array if reached max number */ + if ((nfds == (int) poll_set->max_events) && + (poll_set->max_events * 2 <= HG_POLL_MAX_EVENTS)) { + poll_set->events = realloc(poll_set->events, + sizeof(*poll_set->events) * poll_set->max_events * 2); + HG_UTIL_CHECK_ERROR(!poll_set->events, done, ret, HG_UTIL_FAIL, + "realloc() failed (%s)", strerror(errno)); + + poll_set->max_events *= 2; + } #elif defined(HG_UTIL_HAS_SYSEVENT_H) - struct kevent poll_events[HG_POLL_MAX_EVENTS]; - struct timespec timeout_spec; - ldiv_t ld; - - /* Get sec / nsec */ - ld = ldiv(timeout, 1000L); - timeout_spec.tv_sec = ld.quot; - timeout_spec.tv_nsec = ld.rem * 1000000L; - - nfds = kevent( - poll_set->fd, NULL, 0, poll_events, max_events, &timeout_spec); - HG_UTIL_CHECK_ERROR(nfds == -1 && errno != EINTR, done, ret, - HG_UTIL_FAIL, "kevent() failed (%s)", strerror(errno)); - - for (i = 0; i < nfds; ++i) { - struct hg_poll_data *hg_poll_data = - (struct hg_poll_data *) poll_events[i].udata; - int rc; - - HG_UTIL_CHECK_ERROR(hg_poll_data == NULL, done, ret, HG_UTIL_FAIL, - "NULL poll data"); - - if (!hg_poll_data->poll_cb) - continue; - - rc = hg_poll_data->poll_cb(hg_poll_data->poll_arg, 0, &events[i]); - HG_UTIL_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, HG_UTIL_FAIL, - "poll cb failed"); - } -#else - struct pollfd poll_events[HG_POLL_MAX_EVENTS] = {0}; - struct hg_poll_data *poll_data_events[HG_POLL_MAX_EVENTS] = {NULL}; - struct hg_poll_data *hg_poll_data = NULL; - int nevents = 0; - - /* Reset revents */ - hg_thread_spin_lock(&poll_set->poll_data_list_lock); - for (hg_poll_data = HG_LIST_FIRST(&poll_set->poll_data_list); - hg_poll_data && (nevents < max_poll_events); - hg_poll_data = HG_LIST_NEXT(hg_poll_data, entry), nevents++) { - poll_events[nevents] = hg_poll_data->pollfd; - poll_data_events[nevents] = hg_poll_data; - } - hg_thread_spin_unlock(&poll_set->poll_data_list_lock); + struct timespec timeout_spec; + ldiv_t ld; - nfds = poll(poll_events, nevents, (int) timeout); - HG_UTIL_CHECK_ERROR(nfds == -1 && errno != EINTR, done, ret, - HG_UTIL_FAIL, "poll() failed (%s)", strerror(errno)); + /* Get sec / nsec */ + ld = ldiv(timeout, 1000L); + timeout_spec.tv_sec = ld.quot; + timeout_spec.tv_nsec = ld.rem * 1000000L; - /* An event on one of the fds has occurred. */ - for (i = 0; i < nfds; ++i) { - int rc; + nfds = kevent( + poll_set->fd, NULL, 0, poll_set->events, max_events, &timeout_spec); + HG_UTIL_CHECK_ERROR(nfds == -1 && errno != EINTR, done, ret, HG_UTIL_FAIL, + "kevent() failed (%s)", strerror(errno)); - if (!(poll_events[i].revents & poll_events[i].events)) - continue; + for (i = 0; i < nfds; ++i) { + events[i].events = 0; + events[i].data.ptr = (hg_util_uint64_t) poll_set->events[i].udata; - /* TODO check POLLHUP | POLLERR | POLLNVAL */ - if (!poll_data_events[i]->poll_cb) - continue; + if (poll_set->events[i].flags & EVFILT_READ) + events[i].events |= HG_POLLIN; - rc = poll_data_events[i]->poll_cb( - poll_data_events[i]->poll_arg, 0, &events[i]); - HG_UTIL_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, HG_UTIL_FAIL, - "poll cb failed"); - } + if (poll_set->events[i].flags & EVFILT_WRITE) + events[i].events |= HG_POLLOUT; + } - if (nfds) { - /* TODO should figure where to call hg_event_get() */ - int rc = hg_event_set(poll_set->fd); - HG_UTIL_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, HG_UTIL_FAIL, - "hg_event_set() failed (%s)", strerror(errno)); - } -#endif - } else { -#ifdef _WIN32 + /* Grow array if reached max number */ + if ((nfds == (int) poll_set->max_events) && + (poll_set->max_events * 2 <= HG_POLL_MAX_EVENTS)) { + poll_set->events = realloc(poll_set->events, + sizeof(*poll_set->events) * poll_set->max_events * 2); + HG_UTIL_CHECK_ERROR(!poll_set->events, done, ret, HG_UTIL_FAIL, + "realloc() failed (%s)", strerror(errno)); + poll_set->max_events *= 2; + } #else - struct hg_poll_data *poll_data_events[HG_POLL_MAX_EVENTS] = {NULL}; - struct hg_poll_data *hg_poll_data; - int nevents = 0; - - /* Reset revents */ - hg_thread_spin_lock(&poll_set->poll_data_list_lock); - for (hg_poll_data = HG_LIST_FIRST(&poll_set->poll_data_list); - hg_poll_data && (nevents < max_poll_events); - hg_poll_data = HG_LIST_NEXT(hg_poll_data, entry), nevents++) - poll_data_events[nevents] = hg_poll_data; - hg_thread_spin_unlock(&poll_set->poll_data_list_lock); - - nfds = nevents; - for (i = 0; i < nfds; ++i) { - int rc; - - if (!poll_data_events[i]->poll_cb) - continue; - - rc = poll_data_events[i]->poll_cb( - poll_data_events[i]->poll_arg, 0, &events[i]); - HG_UTIL_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, HG_UTIL_FAIL, - "poll cb failed"); - } -#endif + int nevent = 0, rc; + hg_util_bool_t signaled; + + rc = hg_event_get(poll_set->fd, &signaled); + HG_UTIL_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, HG_UTIL_FAIL, + "hg_event_get() failed (%s)", strerror(errno)); + if (signaled) { + /* Should we do anything in that case? */ + } + + hg_thread_mutex_lock(&poll_set->lock); + + /* Reset revents */ + for (i = 0; i < (int) poll_set->nfds; i++) + poll_set->events[i].revents = 0; + + nfds = poll(poll_set->events, (nfds_t) poll_set->nfds, (int) timeout); + HG_UTIL_CHECK_ERROR(nfds == -1 && errno != EINTR, unlock, ret, HG_UTIL_FAIL, + "poll() failed (%s)", strerror(errno)); + + nfds = (int) MIN(max_poll_events, nfds); + + /* An event on one of the fds has occurred. */ + for (i = 0; i < (int) poll_set->nfds && nevent < nfds; ++i) { + events[i].events = 0; + events[i].data.u64 = (hg_util_uint64_t) poll_set->event_data[i].u64; + + if (poll_set->events[i].revents & POLLIN) + events[i].events |= HG_POLLIN; + + if (poll_set->events[i].revents & POLLOUT) + events[i].events |= HG_POLLOUT; + + /* Don't change the if/else order */ + if (poll_set->events[i].revents & POLLERR) + events[i].events |= HG_POLLERR; + else if (poll_set->events[i].revents & POLLHUP) + events[i].events |= HG_POLLHUP; + else if (poll_set->events[i].events & POLLNVAL) + events[i].events |= HG_POLLERR; + + nevent++; + } + + hg_thread_mutex_unlock(&poll_set->lock); + + HG_UTIL_CHECK_ERROR(nevent != nfds, done, ret, HG_UTIL_FAIL, + "found only %d events, expected %d", nevent, nfds); + + if (nfds > 0) { + /* TODO should figure where to call hg_event_get() */ + rc = hg_event_set(poll_set->fd); + HG_UTIL_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, HG_UTIL_FAIL, + "hg_event_set() failed (%s)", strerror(errno)); } +#endif if (actual_events) *actual_events = (unsigned int) nfds; done: return ret; + +#if !defined(_WIN32) && !defined(HG_UTIL_HAS_SYSEPOLL_H) && \ + !defined(HG_UTIL_HAS_SYSEVENT_H) +unlock: + hg_thread_mutex_unlock(&poll_set->lock); + + return ret; +#endif } diff --git a/src/util/mercury_poll.h b/src/util/mercury_poll.h index 8922f378..74761282 100644 --- a/src/util/mercury_poll.h +++ b/src/util/mercury_poll.h @@ -13,44 +13,24 @@ #include "mercury_util_config.h" -/** - * Purpose: define an interface that either polls or allows busy wait - * without entering system calls. - */ - /*************************************/ /* Public Type and Struct Definition */ /*************************************/ typedef struct hg_poll_set hg_poll_set_t; +typedef union hg_poll_data { + void *ptr; + int fd; + hg_util_uint32_t u32; + hg_util_uint64_t u64; +} hg_poll_data_t; + struct hg_poll_event { - hg_util_bool_t progressed; /* Indicates progress */ - void *ptr; /* Pointer to user data */ + hg_util_uint32_t events; /* Poll events */ + hg_poll_data_t data; /* User data variable */ }; -/** - * Callback that can be used to signal when it is safe to block on the - * poll set or if blocking could hang the application. - * - * \param arg [IN] function argument - * - * \return HG_UTIL_TRUE if it is safe to block or HG_UTIL_FALSE otherwise - */ -typedef hg_util_bool_t (*hg_poll_try_wait_cb_t)(void *arg); - -/** - * Polling callback, arg can be used to pass user arguments, event can be used - * to return user arguments back to hg_poll_wait. - * - * \param arg [IN] pointer to user data - * \param error [IN] any error event has occurred - * \param ptr [OUT] event data output - * - * \return Non-negative on success or negative on failure - */ -typedef int (*hg_poll_cb_t)(void *arg, int error, struct hg_poll_event *event); - /*****************/ /* Public Macros */ /*****************/ @@ -58,8 +38,10 @@ typedef int (*hg_poll_cb_t)(void *arg, int error, struct hg_poll_event *event); /** * Polling events. */ -#define HG_POLLIN 0x001 /* Ready to read. */ -#define HG_POLLOUT 0x004 /* Ready to write. */ +#define HG_POLLIN 0x001 /* There is data to read. */ +#define HG_POLLOUT 0x004 /* Writing now will not block. */ +#define HG_POLLERR 0x008 /* Error condition. */ +#define HG_POLLHUP 0x010 /* Hung up. */ /*********************/ /* Public Prototypes */ @@ -80,7 +62,7 @@ hg_poll_create(void); /** * Destroy a poll set. * - * \param poll_set [IN] pointer to poll set + * \param poll_set [IN/OUT] pointer to poll set * * \return Non-negative on success or negative on failure */ @@ -97,35 +79,17 @@ hg_poll_destroy(hg_poll_set_t *poll_set); HG_UTIL_PUBLIC int hg_poll_get_fd(hg_poll_set_t *poll_set); -/** - * Set a callback that can be used to signal when it is safe to block on the - * poll set or if blocking could hang the application, in which case behavior - * is the same as passing a timeout of 0. - * - * \param poll_set [IN] pointer to poll set - * \param try_wait_cb [IN] function pointer - * \param try_wait_arg [IN] function pointer argument - * - * \return Non-negative on success or negative on failure - */ -HG_UTIL_PUBLIC int -hg_poll_set_try_wait(hg_poll_set_t *poll_set, hg_poll_try_wait_cb_t try_wait_cb, - void *try_wait_arg); - /** * Add file descriptor to poll set. * * \param poll_set [IN] pointer to poll set * \param fd [IN] file descriptor - * \param flags [IN] polling flags (HG_POLLIN, etc) - * \param poll_cb [IN] function pointer - * \param poll_cb_args [IN] function pointer argument + * \param event [IN] pointer to event struct * * \return Non-negative on success or negative on failure */ HG_UTIL_PUBLIC int -hg_poll_add(hg_poll_set_t *poll_set, int fd, unsigned int flags, - hg_poll_cb_t poll_cb, void *poll_cb_arg); +hg_poll_add(hg_poll_set_t *poll_set, int fd, struct hg_poll_event *event); /** * Remove file descriptor from poll set. @@ -139,16 +103,13 @@ HG_UTIL_PUBLIC int hg_poll_remove(hg_poll_set_t *poll_set, int fd); /** - * Wait on a poll set for timeout ms, progressed indicating whether progress has - * been made after that call returns. If timeout is 0, progress is performed - * on all the registered polling callbacks and hg_poll_wait() exits as soon as - * progress is made. If timeout is non 0, the system dependent polling function - * call is entered and progress is performed on the list of file descriptors - * for which an event has occurred. + * Wait on a poll set for timeout ms, and return at most max_events. * * \param poll_set [IN] pointer to poll set * \param timeout [IN] timeout (in milliseconds) - * \param progressed [OUT] pointer to boolean indicating progress made + * \param max_events [IN] max number of events + * \param events [IN/OUT] array of events to be returned + * \param actual_events [OUT] actual number of events returned * * \return Non-negative on success or negative on failure */ From ea49057a4a0fee88b29eca60a4a925ab1289d66a Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Thu, 18 Jun 2020 13:31:13 -0500 Subject: [PATCH 22/30] NA SM: update progress mechanism after mercury poll updates --- src/na/na_sm.c | 284 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 184 insertions(+), 100 deletions(-) diff --git a/src/na/na_sm.c b/src/na/na_sm.c index 5fc2b9d5..fdaedb11 100644 --- a/src/na/na_sm.c +++ b/src/na/na_sm.c @@ -100,6 +100,8 @@ /* Private data access */ #define NA_SM_CLASS(na_class) ((struct na_sm_class *) (na_class->plugin_class)) +#define NA_SM_CONTEXT(context) \ + ((struct na_sm_context *) (context->plugin_context)) /* Generate SHM file name */ #define NA_SM_GEN_SHM_NAME(filename, maxlen, username, pid, id) \ @@ -147,8 +149,8 @@ typedef union { /* Msg buffers (page aligned) */ struct na_sm_copy_buf { - char buf[NA_SM_NUM_BUFS][NA_SM_COPY_BUF_SIZE]; /* Array of buffers */ hg_thread_spin_t buf_locks[NA_SM_NUM_BUFS]; /* Locks on buffers */ + char buf[NA_SM_NUM_BUFS][NA_SM_COPY_BUF_SIZE]; /* Array of buffers */ na_sm_cacheline_atomic_int64_t available; /* Available bitmask */ }; @@ -215,6 +217,13 @@ struct na_sm_region { na_sm_cacheline_atomic_int256_t available; /* Available pairs */ }; +/* Poll type */ +typedef enum na_sm_poll_type { + NA_SM_POLL_SOCK = 1, + NA_SM_POLL_RX_NOTIFY, + NA_SM_POLL_TX_NOTIFY +} na_sm_poll_type_t; + /* Address */ struct na_sm_addr { HG_LIST_ENTRY(na_sm_addr) entry; /* Entry in poll list */ @@ -223,6 +232,8 @@ struct na_sm_addr { struct na_sm_msg_queue *rx_queue; /* Pointer to shared rx queue */ int tx_notify; /* Notify fd for tx queue */ int rx_notify; /* Notify fd for rx queue */ + na_sm_poll_type_t tx_poll_type; /* Tx poll type */ + na_sm_poll_type_t rx_poll_type; /* Rx poll type */ hg_atomic_int32_t ref_count; /* Ref count */ pid_t pid; /* PID */ na_uint8_t id; /* SM ID */ @@ -316,9 +327,15 @@ struct na_sm_endpoint { struct na_sm_addr *source_addr; /* Source addr */ hg_poll_set_t *poll_set; /* Poll set */ int sock; /* Sock fd */ + na_sm_poll_type_t sock_poll_type; /* Sock poll type */ na_bool_t listen; /* Listen on sock */ }; +/* Private context */ +struct na_sm_context { + struct hg_poll_event events[NA_SM_MAX_EVENTS]; +}; + /* Private data */ struct na_sm_class { struct na_sm_endpoint endpoint; /* Endpoint */ @@ -517,8 +534,7 @@ na_sm_event_get(int event, na_bool_t *signaled); * Register addr to poll set. */ static na_return_t -na_sm_poll_register( - hg_poll_set_t *poll_set, int fd, hg_poll_cb_t poll_cb, void *poll_arg); +na_sm_poll_register(hg_poll_set_t *poll_set, int fd, void *ptr); /** * Deregister addr from poll set. @@ -639,8 +655,9 @@ na_sm_offset_translate(struct na_sm_mem_handle *mem_handle, na_offset_t offset, /** * Progress on endpoint sock. */ -static int -na_sm_progress_sock(void *arg, int error, struct hg_poll_event *event); +static na_return_t +na_sm_progress_sock(struct na_sm_endpoint *na_sm_endpoint, const char *username, + na_bool_t *progressed); /** * Process cmd. @@ -652,14 +669,14 @@ na_sm_process_cmd(struct na_sm_endpoint *na_sm_endpoint, const char *username, /** * Progress on tx notifications. */ -static int -na_sm_progress_tx_notify(void *arg, int error, struct hg_poll_event *event); +static na_return_t +na_sm_progress_tx_notify(struct na_sm_addr *poll_addr, na_bool_t *progressed); /** * Progress on rx notifications. */ -static int -na_sm_progress_rx_notify(void *arg, int error, struct hg_poll_event *event); +static na_return_t +na_sm_progress_rx_notify(struct na_sm_addr *poll_addr, na_bool_t *progressed); /** * Progress rx queue. @@ -714,6 +731,14 @@ na_sm_initialize( static na_return_t na_sm_finalize(na_class_t *na_class); +/* context_create */ +static na_return_t +na_sm_context_create(na_class_t *na_class, void **context, na_uint8_t id); + +/* context_destroy */ +static na_return_t +na_sm_context_destroy(na_class_t *na_class, void *context); + /* cleanup */ static void na_sm_cleanup(void); @@ -883,8 +908,8 @@ const struct na_class_ops NA_PLUGIN_OPS(sm) = { na_sm_initialize, /* initialize */ na_sm_finalize, /* finalize */ na_sm_cleanup, /* cleanup */ - NULL, /* context_create */ - NULL, /* context_destroy */ + na_sm_context_create, /* context_create */ + na_sm_context_destroy, /* context_destroy */ na_sm_op_create, /* op_create */ na_sm_op_destroy, /* op_destroy */ na_sm_addr_lookup, /* addr_lookup */ @@ -1024,8 +1049,8 @@ NA_SM_String_to_host_id(const char *string, na_sm_id_t *id) #else na_return_t ret = NA_SUCCESS; int rc = sscanf(string, "%ld", id); - NA_CHECK_ERROR(rc != 1, done, ret, NA_PROTOCOL_ERROR, - "sscanf() failed, rc: %d", rc); + NA_CHECK_ERROR( + rc != 1, done, ret, NA_PROTOCOL_ERROR, "sscanf() failed, rc: %d", rc); done: return ret; @@ -1757,14 +1782,13 @@ na_sm_event_get(int event, na_bool_t *signaled) /*---------------------------------------------------------------------------*/ static na_return_t -na_sm_poll_register( - hg_poll_set_t *poll_set, int fd, hg_poll_cb_t poll_cb, void *poll_arg) +na_sm_poll_register(hg_poll_set_t *poll_set, int fd, void *ptr) { - unsigned int flags = HG_POLLIN; + struct hg_poll_event event = {.events = HG_POLLIN, .data.ptr = ptr}; na_return_t ret = NA_SUCCESS; int rc; - rc = hg_poll_add(poll_set, fd, flags, poll_cb, poll_arg); + rc = hg_poll_add(poll_set, fd, &event); NA_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, na_sm_errno_to_na(errno), "hg_poll_add() failed"); @@ -1849,11 +1873,12 @@ na_sm_endpoint_open(struct na_sm_endpoint *na_sm_endpoint, const char *username, NA_CHECK_NA_ERROR(error, ret, "Could not open sock"); if (listen) { + na_sm_endpoint->sock_poll_type = NA_SM_POLL_SOCK; NA_LOG_DEBUG( "Registering sock %d for polling", na_sm_endpoint->sock); /* Add sock to poll set (ony required if we're listening) */ ret = na_sm_poll_register(na_sm_endpoint->poll_set, - na_sm_endpoint->sock, na_sm_progress_sock, na_sm_endpoint); + na_sm_endpoint->sock, &na_sm_endpoint->sock_poll_type); NA_CHECK_NA_ERROR(error, ret, "Could not add sock to poll set"); sock_registered = NA_TRUE; } @@ -1873,9 +1898,10 @@ na_sm_endpoint_open(struct na_sm_endpoint *na_sm_endpoint, const char *username, /* Add source tx notify to poll set for local notifications */ if (!no_wait) { + na_sm_endpoint->source_addr->tx_poll_type = NA_SM_POLL_TX_NOTIFY; NA_LOG_DEBUG("Registering tx notify %d for polling", tx_notify); ret = na_sm_poll_register(na_sm_endpoint->poll_set, tx_notify, - na_sm_progress_tx_notify, na_sm_endpoint->source_addr); + &na_sm_endpoint->source_addr->tx_poll_type); NA_CHECK_NA_ERROR(error, ret, "Could not add tx notify to poll set"); } @@ -2328,11 +2354,12 @@ na_sm_addr_create(struct na_sm_endpoint *na_sm_endpoint, } if (na_sm_endpoint->poll_set && (na_sm_addr->rx_notify > 0)) { + na_sm_addr->rx_poll_type = NA_SM_POLL_RX_NOTIFY; NA_LOG_DEBUG( "Registering rx notify %d for polling", na_sm_addr->rx_notify); /* Add remote rx notify to poll set */ ret = na_sm_poll_register(na_sm_endpoint->poll_set, - na_sm_addr->rx_notify, na_sm_progress_rx_notify, na_sm_addr); + na_sm_addr->rx_notify, &na_sm_addr->rx_poll_type); NA_CHECK_NA_ERROR(done, ret, "Could not add rx notify to poll set"); } @@ -2655,38 +2682,28 @@ na_sm_offset_translate(struct na_sm_mem_handle *mem_handle, na_offset_t offset, } /*---------------------------------------------------------------------------*/ -static int -na_sm_progress_sock(void *arg, int error, struct hg_poll_event *event) +static na_return_t +na_sm_progress_sock(struct na_sm_endpoint *na_sm_endpoint, const char *username, + na_bool_t *progressed) { - struct na_sm_endpoint *na_sm_endpoint = (struct na_sm_endpoint *) arg; - struct na_sm_class *na_sm_class = - container_of(na_sm_endpoint, struct na_sm_class, endpoint); na_sm_cmd_hdr_t cmd_hdr = {.val = 0}; - int tx_notify, rx_notify; - na_bool_t progressed = NA_FALSE; + int tx_notify = -1, rx_notify = -1; na_return_t ret = NA_SUCCESS; - NA_CHECK_ERROR(error, done, ret, NA_FAULT, "Unexpected poll error"); - /* Attempt to receive addr info (events, queue index) */ ret = na_sm_addr_event_recv( - na_sm_endpoint->sock, &cmd_hdr, &tx_notify, &rx_notify, &progressed); + na_sm_endpoint->sock, &cmd_hdr, &tx_notify, &rx_notify, progressed); NA_CHECK_NA_ERROR(done, ret, "Could not recv addr events"); - if (!progressed) { - event->progressed = HG_UTIL_FALSE; - goto done; - } - - /* Process received cmd, TODO would be nice to use cmd queue */ - ret = na_sm_process_cmd( - na_sm_endpoint, na_sm_class->username, cmd_hdr, tx_notify, rx_notify); - NA_CHECK_NA_ERROR(done, ret, "Could not process cmd"); - event->progressed = HG_UTIL_TRUE; - event->ptr = NULL; + if (*progressed) { + /* Process received cmd, TODO would be nice to use cmd queue */ + ret = na_sm_process_cmd( + na_sm_endpoint, username, cmd_hdr, tx_notify, rx_notify); + NA_CHECK_NA_ERROR(done, ret, "Could not process cmd"); + } done: - return (ret == NA_SUCCESS) ? HG_UTIL_SUCCESS : HG_UTIL_FAIL; + return ret; } /*---------------------------------------------------------------------------*/ @@ -2773,51 +2790,37 @@ na_sm_process_cmd(struct na_sm_endpoint *na_sm_endpoint, const char *username, } /*---------------------------------------------------------------------------*/ -static int -na_sm_progress_tx_notify(void *arg, int error, struct hg_poll_event *event) +static na_return_t +na_sm_progress_tx_notify(struct na_sm_addr *poll_addr, na_bool_t *progressed) { - struct na_sm_addr *poll_addr = (struct na_sm_addr *) arg; na_return_t ret = NA_SUCCESS; int rc; - NA_CHECK_ERROR(error, done, ret, NA_FAULT, "Unexpected poll error"); - /* Local notification only */ - rc = hg_event_get(poll_addr->tx_notify, &event->progressed); + rc = hg_event_get(poll_addr->tx_notify, (hg_util_bool_t *) progressed); NA_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, na_sm_errno_to_na(errno), "Could not get completion notification"); - if (!event->progressed) - goto done; NA_LOG_DEBUG("Progressed tx notify %d", poll_addr->tx_notify); - event->ptr = NULL; - done: - return (ret == NA_SUCCESS) ? HG_UTIL_SUCCESS : HG_UTIL_FAIL; + return ret; } /*---------------------------------------------------------------------------*/ -static int -na_sm_progress_rx_notify(void *arg, int error, struct hg_poll_event *event) +static na_return_t +na_sm_progress_rx_notify(struct na_sm_addr *poll_addr, na_bool_t *progressed) { - struct na_sm_addr *poll_addr = (struct na_sm_addr *) arg; na_return_t ret = NA_SUCCESS; - NA_CHECK_ERROR(error, done, ret, NA_FAULT, "Unexpected poll error"); - /* Remote notification only */ - ret = na_sm_event_get(poll_addr->rx_notify, &event->progressed); + ret = na_sm_event_get(poll_addr->rx_notify, progressed); NA_CHECK_NA_ERROR(done, ret, "Could not get completion notification"); - if (!event->progressed) - goto done; NA_LOG_DEBUG("Progressed rx notify %d", poll_addr->rx_notify); - event->ptr = poll_addr; - done: - return (ret == NA_SUCCESS) ? HG_UTIL_SUCCESS : HG_UTIL_FAIL; + return ret; } /*---------------------------------------------------------------------------*/ @@ -3217,7 +3220,7 @@ na_sm_initialize(na_class_t *na_class, const struct na_info NA_UNUSED *na_info, /* Initialize private data */ na_class->plugin_class = malloc(sizeof(struct na_sm_class)); NA_CHECK_ERROR(na_class->plugin_class == NULL, error, ret, NA_NOMEM, - "Could not allocate NA private data class"); + "Could not allocate SM private class"); memset(na_class->plugin_class, 0, sizeof(struct na_sm_class)); NA_SM_CLASS(na_class)->no_wait = no_wait; NA_SM_CLASS(na_class)->max_contexts = max_contexts; @@ -3272,6 +3275,30 @@ na_sm_finalize(na_class_t *na_class) return ret; } +/*---------------------------------------------------------------------------*/ +static na_return_t +na_sm_context_create( + na_class_t NA_UNUSED *na_class, void **context, na_uint8_t NA_UNUSED id) +{ + na_return_t ret = NA_SUCCESS; + + *context = malloc(sizeof(struct na_sm_context)); + NA_CHECK_ERROR(*context == NULL, done, ret, NA_NOMEM, + "Could not allocate SM private context"); + +done: + return ret; +} + +/*---------------------------------------------------------------------------*/ +static na_return_t +na_sm_context_destroy(na_class_t NA_UNUSED *na_class, void *context) +{ + free(context); + + return NA_SUCCESS; +} + /*---------------------------------------------------------------------------*/ static void na_sm_cleanup(void) @@ -4409,26 +4436,19 @@ na_sm_poll_get_fd(na_class_t *na_class, na_context_t NA_UNUSED *context) static NA_INLINE na_bool_t na_sm_poll_try_wait(na_class_t *na_class, na_context_t NA_UNUSED *context) { - struct na_sm_endpoint *na_sm_endpoint = &NA_SM_CLASS(na_class)->endpoint; - struct na_sm_addr_list *poll_addr_list = &na_sm_endpoint->poll_addr_list; struct na_sm_addr *na_sm_addr; /* Check whether something is in one of the rx queues */ - hg_thread_spin_lock(&poll_addr_list->lock); - HG_LIST_FOREACH (na_sm_addr, &poll_addr_list->list, entry) { + hg_thread_spin_lock(&NA_SM_CLASS(na_class)->endpoint.poll_addr_list.lock); + HG_LIST_FOREACH (na_sm_addr, + &NA_SM_CLASS(na_class)->endpoint.poll_addr_list.list, entry) { if (!na_sm_msg_queue_is_empty(na_sm_addr->rx_queue)) { - hg_thread_spin_unlock(&poll_addr_list->lock); + hg_thread_spin_unlock( + &NA_SM_CLASS(na_class)->endpoint.poll_addr_list.lock); return NA_FALSE; } } - hg_thread_spin_unlock(&poll_addr_list->lock); - - /* Check whether something is in the cmd queue */ - if (na_sm_endpoint->source_addr->shared_region && - !na_sm_cmd_queue_is_empty( - &na_sm_endpoint->source_addr->shared_region->cmd_queue)) { - return NA_FALSE; - } + hg_thread_spin_unlock(&NA_SM_CLASS(na_class)->endpoint.poll_addr_list.lock); return NA_TRUE; } @@ -4436,23 +4456,27 @@ na_sm_poll_try_wait(na_class_t *na_class, na_context_t NA_UNUSED *context) /*---------------------------------------------------------------------------*/ static na_return_t na_sm_progress( - na_class_t *na_class, na_context_t NA_UNUSED *context, unsigned int timeout) + na_class_t *na_class, na_context_t *context, unsigned int timeout) { struct na_sm_endpoint *na_sm_endpoint = &NA_SM_CLASS(na_class)->endpoint; + const char *username = NA_SM_CLASS(na_class)->username; + struct hg_poll_event *events = NA_SM_CONTEXT(context)->events; double remaining = timeout / 1000.0; /* Convert timeout in ms into seconds */ na_return_t ret = NA_TIMEOUT; do { - struct hg_poll_event events[NA_SM_MAX_EVENTS] = {0}; - unsigned int nevents = 0, i; na_bool_t progressed = NA_FALSE; hg_time_t t1, t2; if (timeout) hg_time_get_current(&t1); - if (na_sm_endpoint->poll_set) { + if (timeout && na_sm_endpoint->poll_set) { + unsigned int nevents = 0, i; + /* Just wait on a single event, anything greater may increase + * latency, and slow down progress, we will not wait next round + * if something is still in the queues */ int rc = hg_poll_wait(na_sm_endpoint->poll_set, (unsigned int) (remaining * 1000.0), NA_SM_MAX_EVENTS, events, &nevents); @@ -4461,43 +4485,103 @@ na_sm_progress( /* Process events */ for (i = 0; i < nevents; i++) { - if (events[i].progressed && events[i].ptr) { - na_bool_t progressed_rx; - - ret = na_sm_progress_rx_queue( - na_sm_endpoint, events[i].ptr, &progressed_rx); - NA_CHECK_NA_ERROR(done, ret, "Could not progress rx queue"); - progressed |= progressed_rx; + struct na_sm_addr *poll_addr = NULL; + na_bool_t progressed_notify = NA_FALSE; + na_bool_t progressed_rx = NA_FALSE; + + switch (*(na_sm_poll_type_t *) events[i].data.ptr) { + case NA_SM_POLL_SOCK: + NA_LOG_DEBUG("NA_SM_POLL_SOCK event"); + ret = na_sm_progress_sock( + na_sm_endpoint, username, &progressed_notify); + NA_CHECK_NA_ERROR(done, ret, "Could not progress sock"); + break; + case NA_SM_POLL_TX_NOTIFY: + NA_LOG_DEBUG("NA_SM_POLL_TX_NOTIFY event"); + poll_addr = container_of(events[i].data.ptr, + struct na_sm_addr, tx_poll_type); + ret = na_sm_progress_tx_notify( + poll_addr, &progressed_notify); + NA_CHECK_NA_ERROR( + done, ret, "Could not progress tx notify"); + break; + case NA_SM_POLL_RX_NOTIFY: + NA_LOG_DEBUG("NA_SM_POLL_RX_NOTIFY event"); + poll_addr = container_of(events[i].data.ptr, + struct na_sm_addr, rx_poll_type); + + ret = na_sm_progress_rx_notify( + poll_addr, &progressed_notify); + NA_CHECK_NA_ERROR( + done, ret, "Could not progress rx notify"); + + ret = na_sm_progress_rx_queue( + na_sm_endpoint, poll_addr, &progressed_rx); + NA_CHECK_NA_ERROR( + done, ret, "Could not progress rx queue"); + + break; + default: + NA_GOTO_ERROR(done, ret, NA_INVALID_ARG, + "Operation type %d not supported", + *(na_sm_poll_type_t *) events[i].data.ptr); } - progressed |= events[i].progressed; + + progressed |= (progressed_rx | progressed_notify); } } else { struct na_sm_addr_list *poll_addr_list = &na_sm_endpoint->poll_addr_list; - struct na_sm_addr *na_sm_addr; + struct na_sm_addr *poll_addr; /* Check whether something is in one of the rx queues */ hg_thread_spin_lock(&poll_addr_list->lock); - HG_LIST_FOREACH (na_sm_addr, &poll_addr_list->list, entry) { - na_bool_t progressed_rx; - + HG_LIST_FOREACH (poll_addr, &poll_addr_list->list, entry) { + na_bool_t progressed_rx = NA_FALSE; + + hg_thread_spin_unlock(&poll_addr_list->lock); + + if (na_sm_endpoint->poll_set) { + na_bool_t progressed_notify = NA_FALSE; + ret = + na_sm_progress_rx_notify(poll_addr, &progressed_notify); + NA_CHECK_NA_ERROR( + done, ret, "Could not progress rx notify"); + progressed |= progressed_notify; + } ret = na_sm_progress_rx_queue( - na_sm_endpoint, na_sm_addr, &progressed_rx); + na_sm_endpoint, poll_addr, &progressed_rx); NA_CHECK_NA_ERROR(done, ret, "Could not progress rx queue"); - progressed |= progressed_rx; + + hg_thread_spin_lock(&poll_addr_list->lock); } hg_thread_spin_unlock(&poll_addr_list->lock); /* Look for message in cmd queue (if listening) */ - if (na_sm_endpoint->source_addr->shared_region) { + if (na_sm_endpoint->poll_set) { + na_bool_t progressed_notify = NA_FALSE; + + ret = na_sm_progress_tx_notify( + na_sm_endpoint->source_addr, &progressed_notify); + NA_CHECK_NA_ERROR(done, ret, "Could not progress tx notify"); + progressed |= progressed_notify; + + if (na_sm_endpoint->source_addr->shared_region) { + na_bool_t progressed_sock = NA_FALSE; + ret = na_sm_progress_sock( + na_sm_endpoint, username, &progressed_sock); + NA_CHECK_NA_ERROR(done, ret, "Could not progress sock"); + progressed |= progressed_sock; + } + } else if (na_sm_endpoint->source_addr->shared_region) { na_sm_cmd_hdr_t cmd_hdr = {.val = 0}; while (na_sm_cmd_queue_pop( &na_sm_endpoint->source_addr->shared_region->cmd_queue, &cmd_hdr)) { - ret = na_sm_process_cmd(na_sm_endpoint, - NA_SM_CLASS(na_class)->username, cmd_hdr, -1, -1); + ret = na_sm_process_cmd( + na_sm_endpoint, username, cmd_hdr, -1, -1); NA_CHECK_NA_ERROR(done, ret, "Could not process cmd"); progressed |= NA_TRUE; } @@ -4513,7 +4597,7 @@ na_sm_progress( remaining -= hg_time_to_double(hg_time_subtract(t2, t1)); } - if (nevents == 0 || !progressed) + if (!progressed) ret = NA_TIMEOUT; /* Return NA_TIMEOUT if no events */ } while (remaining > 0 && (ret != NA_SUCCESS)); From bdb6e6e22b652ac9dbff145ac8b0ee4d1d2aeb2c Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Thu, 18 Jun 2020 13:33:52 -0500 Subject: [PATCH 23/30] HG Core: update progress mechanism after mercury poll updates Unify progress routines (fix #252) --- src/mercury_core.c | 540 +++++++++++++++++++-------------------------- 1 file changed, 227 insertions(+), 313 deletions(-) diff --git a/src/mercury_core.c b/src/mercury_core.c index a1030823..75d596f0 100644 --- a/src/mercury_core.c +++ b/src/mercury_core.c @@ -41,7 +41,7 @@ #define HG_CORE_ATOMIC_QUEUE_SIZE 1024 #define HG_CORE_PENDING_INCR 256 #define HG_CORE_CLEANUP_TIMEOUT 1000 -#define HG_CORE_MAX_EVENTS 16 +#define HG_CORE_MAX_EVENTS 1 #define HG_CORE_MAX_TRIGGER_COUNT 1 #ifdef HG_HAS_SM_ROUTING # define HG_CORE_ADDR_MAX_SIZE 256 @@ -107,6 +107,15 @@ struct hg_core_private_class { #endif }; +/* Poll type */ +typedef enum hg_core_poll_type { + HG_CORE_POLL_LOOPBACK = 1, +#ifdef HG_HAS_SM_ROUTING + HG_CORE_POLL_SM, +#endif + HG_CORE_POLL_NA +} hg_core_poll_type_t; + /* HG context */ struct hg_core_private_context { struct hg_core_context core_context; /* Must remain as first field */ @@ -122,8 +131,7 @@ struct hg_core_private_context { hg_return_t (*handle_create)(hg_core_handle_t, void *); /* handle_create */ void *handle_create_arg; /* handle_create arg */ struct hg_poll_set *poll_set; /* Context poll set */ - hg_return_t (*progress)(struct hg_core_private_context *context, - unsigned int timeout); /* Progress function */ + struct hg_poll_event poll_events[HG_CORE_MAX_EVENTS]; /* Context poll events */ hg_atomic_int32_t backfill_queue_count; /* Backfill queue count */ hg_atomic_int32_t trigger_waiting; /* Waiting in trigger */ hg_atomic_int32_t n_handles; /* Atomic used for number of handles */ @@ -638,7 +646,8 @@ hg_core_reset_post( */ static hg_return_t hg_core_progress_na( - struct hg_core_private_context *context, + na_class_t *na_class, + na_context_t *na_context, unsigned int timeout ); @@ -646,49 +655,25 @@ hg_core_progress_na( /** * Completion queue notification callback. */ -static HG_INLINE int -hg_core_completion_queue_notify_cb( - void *arg, - int error, - struct hg_poll_event *event - ); -#endif - -/** - * Progress callback on NA layer when hg_core_progress_poll() is used. - */ -static int -hg_core_progress_na_cb( - void *arg, - int error, - struct hg_poll_event *event - ); - -#ifdef HG_HAS_SM_ROUTING -/** - * Progress callback on NA SM layer when hg_core_progress_poll() is used. - */ -static int -hg_core_progress_na_sm_cb( - void *arg, - int error, - struct hg_poll_event *event +static HG_INLINE hg_return_t +hg_core_progress_loopback_notify( + struct hg_core_private_context *context ); #endif /** - * Callback for HG poll progress that determines when it is safe to block. + * Determines when it is safe to block. */ -static HG_INLINE hg_util_bool_t -hg_core_poll_try_wait_cb( - void *arg +static HG_INLINE hg_bool_t +hg_core_poll_try_wait( + struct hg_core_private_context *context ); /** * Make progress. */ static hg_return_t -hg_core_progress_poll( +hg_core_progress( struct hg_core_private_context *context, unsigned int timeout ); @@ -941,7 +926,7 @@ hg_core_context_lists_wait(struct hg_core_private_context *context) if (created_list_empty && pending_list_empty && sm_pending_list_empty) break; - progress_ret = context->progress(context, + progress_ret = hg_core_progress(context, (unsigned int) (remaining * 1000.0)); HG_CHECK_ERROR(progress_ret != HG_SUCCESS && progress_ret != HG_TIMEOUT, done, ret, progress_ret, "Could not make progress"); @@ -2703,164 +2688,17 @@ hg_core_reset_post(struct hg_core_private_handle *hg_core_handle) return ret; } -/*---------------------------------------------------------------------------*/ -#ifdef HG_HAS_SELF_FORWARD -static HG_INLINE int -hg_core_completion_queue_notify_cb(void *arg, - int HG_UNUSED error, struct hg_poll_event *event) -{ - struct hg_core_private_context *context = - (struct hg_core_private_context *) arg; - hg_util_bool_t notified = HG_UTIL_FALSE; - int rc = HG_UTIL_SUCCESS; - - if (!(HG_CORE_CONTEXT_CLASS(context)->progress_mode & NA_NO_BLOCK)) { - /* TODO could prevent from self notifying if hg_poll_wait() not entered */ - rc = hg_event_get(context->completion_queue_notify, ¬ified); - HG_CHECK_ERROR_NORET(rc != HG_UTIL_SUCCESS, done, - "Could not get completion notification"); - } - - if (notified || !hg_atomic_queue_is_empty(context->completion_queue) - || hg_atomic_get32(&context->backfill_queue_count)) { - event->progressed = HG_UTIL_TRUE; /* Progressed */ - goto done; - } - - event->progressed = HG_UTIL_FALSE; - -done: - return rc; -} -#endif - -/*---------------------------------------------------------------------------*/ -static int -hg_core_progress_na_cb(void *arg, int HG_UNUSED error, - struct hg_poll_event *event) -{ - struct hg_core_private_context *context = - (struct hg_core_private_context *) arg; - unsigned int actual_count = 0; - unsigned int completed_count = 0; - int cb_ret[HG_CORE_MAX_TRIGGER_COUNT] = {0}; - int rc = HG_UTIL_SUCCESS; - na_return_t na_ret; - - /* Check progress on NA (no need to call try_wait here) */ - na_ret = NA_Progress(HG_CORE_CONTEXT_CLASS(context)->core_class.na_class, - context->core_context.na_context, 0); - if (na_ret == NA_TIMEOUT) { - /* Nothing progressed */ - event->progressed = HG_UTIL_FALSE; - goto done; - } else - HG_CHECK_ERROR(na_ret != NA_SUCCESS, done, rc, HG_UTIL_FAIL, - "Could not make progress on NA (%s)", NA_Error_to_string(na_ret)); - - /* Trigger everything we can from NA, if something completed it will - * be moved to the HG context completion queue */ - do { - unsigned int i; - - na_ret = NA_Trigger(context->core_context.na_context, 0, - HG_CORE_MAX_TRIGGER_COUNT, cb_ret, &actual_count); - - /* Return value of callback is completion count */ - for (i = 0; i < actual_count; i++) - completed_count += (unsigned int) cb_ret[i]; - } while ((na_ret == NA_SUCCESS) && actual_count); - HG_CHECK_ERROR(na_ret != NA_SUCCESS && na_ret != NA_TIMEOUT, done, rc, - HG_UTIL_FAIL, "Could not trigger NA callback (%s)", - NA_Error_to_string(na_ret)); - - /* We can't only verify that the completion queue is not empty, we need - * to check what was added to the completion queue, as the completion queue - * may have been concurrently emptied */ - if (!completed_count && hg_atomic_queue_is_empty(context->completion_queue) - && !hg_atomic_get32(&context->backfill_queue_count)) { - /* Nothing progressed */ - event->progressed = HG_UTIL_FALSE; - goto done; - } - - event->progressed = HG_UTIL_TRUE; - -done: - return rc; -} - -/*---------------------------------------------------------------------------*/ -#ifdef HG_HAS_SM_ROUTING -static int -hg_core_progress_na_sm_cb(void *arg, int HG_UNUSED error, - struct hg_poll_event *event) -{ - struct hg_core_private_context *context = - (struct hg_core_private_context *) arg; - unsigned int actual_count = 0; - unsigned int completed_count = 0; - int cb_ret[HG_CORE_MAX_TRIGGER_COUNT] = {0}; - int rc = HG_UTIL_SUCCESS; - na_return_t na_ret; - - /* Check progress on NA SM (no need to call try_wait here) */ - na_ret = NA_Progress(HG_CORE_CONTEXT_CLASS(context)->core_class.na_sm_class, - context->core_context.na_sm_context, 0); - if (na_ret == NA_TIMEOUT) { - /* Nothing progressed */ - event->progressed = HG_UTIL_FALSE; - goto done; - } else - HG_CHECK_ERROR(na_ret != NA_SUCCESS, done, rc, HG_UTIL_FAIL, - "Could not make progress on NA SM (%s)", - NA_Error_to_string(na_ret)); - - /* Trigger everything we can from NA, if something completed it will - * be moved to the HG context completion queue */ - do { - unsigned int i; - - na_ret = NA_Trigger(context->core_context.na_sm_context, 0, - HG_CORE_MAX_TRIGGER_COUNT, cb_ret, &actual_count); - - /* Return value of callback is completion count */ - for (i = 0; i < actual_count; i++) - completed_count += (unsigned int)cb_ret[i]; - } while ((na_ret == NA_SUCCESS) && actual_count); - HG_CHECK_ERROR(na_ret != NA_SUCCESS && na_ret != NA_TIMEOUT, done, rc, - HG_UTIL_FAIL, "Could not trigger NA SM callback (%s)", - NA_Error_to_string(na_ret)); - - /* We can't only verify that the completion queue is not empty, we need - * to check what was added to the completion queue, as the completion queue - * may have been concurrently emptied */ - if (!completed_count && hg_atomic_queue_is_empty(context->completion_queue) - && !hg_atomic_get32(&context->backfill_queue_count)) { - /* Nothing progressed */ - event->progressed = HG_UTIL_FALSE; - goto done; - } - - event->progressed = HG_UTIL_TRUE; - -done: - return rc; -} -#endif - /*---------------------------------------------------------------------------*/ static hg_return_t -hg_core_progress_na(struct hg_core_private_context *context, +hg_core_progress_na(na_class_t *na_class, na_context_t *na_context, unsigned int timeout) { double remaining = timeout / 1000.0; /* Convert timeout in ms into seconds */ + unsigned int completed_count = 0; hg_return_t ret = HG_TIMEOUT; for (;;) { unsigned int actual_count = 0; - int cb_ret[HG_CORE_MAX_TRIGGER_COUNT] = {0}; - unsigned int completed_count = 0; unsigned int progress_timeout; na_return_t na_ret; hg_time_t t1, t2; @@ -2868,26 +2706,23 @@ hg_core_progress_na(struct hg_core_private_context *context, /* Trigger everything we can from NA, if something completed it will * be moved to the HG context completion queue */ do { + int cb_ret[HG_CORE_MAX_TRIGGER_COUNT] = {0}; unsigned int i; - na_ret = NA_Trigger(context->core_context.na_context, 0, - HG_CORE_MAX_TRIGGER_COUNT, cb_ret, &actual_count); + na_ret = NA_Trigger(na_context, 0, HG_CORE_MAX_TRIGGER_COUNT, + cb_ret, &actual_count); /* Return value of callback is completion count */ for (i = 0; i < actual_count; i++) - completed_count += (unsigned int)cb_ret[i]; + completed_count += (unsigned int) cb_ret[i]; } while ((na_ret == NA_SUCCESS) && actual_count); HG_CHECK_ERROR(na_ret != NA_SUCCESS && na_ret != NA_TIMEOUT, done, ret, (hg_return_t) na_ret, "Could not trigger NA callback (%s)", NA_Error_to_string(na_ret)); - /* We can't only verify that the completion queue is not empty, we need - * to check what was added to the completion queue, as the completion - * queue may have been concurrently emptied */ - if (completed_count - || !hg_atomic_queue_is_empty(context->completion_queue) - || hg_atomic_get32(&context->backfill_queue_count)) { - ret = HG_SUCCESS; /* Progressed */ + /* Progressed */ + if (completed_count) { + ret = HG_SUCCESS; break; } @@ -2898,31 +2733,24 @@ hg_core_progress_na(struct hg_core_private_context *context, hg_time_get_current(&t1); /* Make sure that it is safe to block */ - if (timeout && NA_Poll_try_wait( - HG_CORE_CONTEXT_CLASS(context)->core_class.na_class, - context->core_context.na_context)) + if (timeout && NA_Poll_try_wait(na_class, na_context)) progress_timeout = (unsigned int) (remaining * 1000.0); else progress_timeout = 0; /* Otherwise try to make progress on NA */ - na_ret = NA_Progress( - HG_CORE_CONTEXT_CLASS(context)->core_class.na_class, - context->core_context.na_context, progress_timeout); + na_ret = NA_Progress(na_class, na_context, progress_timeout); + if (na_ret == NA_TIMEOUT && (remaining <= 0)) + break; + else + HG_CHECK_ERROR(na_ret != NA_SUCCESS && na_ret != NA_TIMEOUT, done, ret, + (hg_return_t) na_ret, "Could not make progress on NA (%s)", + NA_Error_to_string(na_ret)); if (timeout) { hg_time_get_current(&t2); remaining -= hg_time_to_double(hg_time_subtract(t2, t1)); } - - /* Trigger NA callbacks and check whether we completed something */ - if (na_ret == NA_SUCCESS) - continue; - else if (na_ret == NA_TIMEOUT && (remaining <= 0)) - break; - else if (na_ret != NA_TIMEOUT) - HG_GOTO_ERROR(done, ret, (hg_return_t) na_ret, - "Could not make NA Progress (%s)", NA_Error_to_string(na_ret)); } done: @@ -2930,39 +2758,53 @@ hg_core_progress_na(struct hg_core_private_context *context, } /*---------------------------------------------------------------------------*/ -static HG_INLINE hg_util_bool_t -hg_core_poll_try_wait_cb(void *arg) +#ifdef HG_HAS_SELF_FORWARD +static HG_INLINE hg_return_t +hg_core_progress_loopback_notify(struct hg_core_private_context *context) { - struct hg_core_private_context *context = - (struct hg_core_private_context *) arg; + hg_util_bool_t progressed = HG_UTIL_FALSE; + hg_return_t ret = HG_SUCCESS; + int rc; + + rc = hg_event_get(context->completion_queue_notify, &progressed); + if (progressed) + ret = HG_SUCCESS; + else + HG_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, HG_PROTOCOL_ERROR, + "Could not get completion notification"); - /* Do not try to wait if NA_NO_BLOCK is set */ - if (HG_CORE_CONTEXT_CLASS(context)->progress_mode & NA_NO_BLOCK) - return NA_FALSE; +done: + return ret; +} +#endif +/*---------------------------------------------------------------------------*/ +static HG_INLINE hg_bool_t +hg_core_poll_try_wait(struct hg_core_private_context *context) +{ + return HG_FALSE; /* Something is in one of the completion queues */ if (!hg_atomic_queue_is_empty(context->completion_queue) || - hg_atomic_get32(&context->backfill_queue_count)) { - return NA_FALSE; - } + (hg_atomic_get32(&context->backfill_queue_count) > 0)) + return HG_FALSE; #ifdef HG_HAS_SM_ROUTING - if (context->core_context.core_class->na_sm_class) { - na_bool_t ret = NA_Poll_try_wait( - context->core_context.core_class->na_sm_class, - context->core_context.na_sm_context); - if (ret) - return ret; - } + if (context->core_context.core_class->na_sm_class + && !NA_Poll_try_wait(context->core_context.core_class->na_sm_class, + context->core_context.na_sm_context)) + return HG_FALSE; #endif - return NA_Poll_try_wait(context->core_context.core_class->na_class, - context->core_context.na_context); + if (!NA_Poll_try_wait(context->core_context.core_class->na_class, + context->core_context.na_context)) + return HG_FALSE; + + return HG_TRUE; } /*---------------------------------------------------------------------------*/ static hg_return_t -hg_core_progress_poll(struct hg_core_private_context *context, +hg_core_progress(struct hg_core_private_context *context, unsigned int timeout) { double remaining = timeout / 1000.0; /* Convert timeout in ms into seconds */ @@ -2970,28 +2812,112 @@ hg_core_progress_poll(struct hg_core_private_context *context, do { hg_time_t t1, t2; - struct hg_poll_event events[HG_CORE_MAX_EVENTS] = {0}; - unsigned int poll_timeout = - (HG_CORE_CONTEXT_CLASS(context)->progress_mode & NA_NO_BLOCK) ? 0 : - (unsigned int) (remaining * 1000.0); - unsigned int nevents, i; - int rc; if (timeout) hg_time_get_current(&t1); - /* Will call hg_core_poll_try_wait_cb if timeout is not 0 */ - rc = hg_poll_wait(context->poll_set, poll_timeout, HG_CORE_MAX_EVENTS, - events, &nevents); - HG_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, HG_PROTOCOL_ERROR, - "hg_poll_wait() failed"); + /* Only enter blocking wait if it is safe to */ + if (!(HG_CORE_CONTEXT_CLASS(context)->progress_mode & NA_NO_BLOCK) + && context->poll_set && timeout && hg_core_poll_try_wait(context)) { + unsigned int i, nevents; + int rc; + + rc = hg_poll_wait(context->poll_set, + (unsigned int) (remaining * 1000.0), HG_CORE_MAX_EVENTS, + context->poll_events, &nevents); + HG_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, HG_PROTOCOL_ERROR, + "hg_poll_wait() failed"); - /* We progressed, return success */ - for (i = 0; i < nevents; i++) { - if (events[i].progressed) { + for (i = 0; i < nevents; i++) { + switch (context->poll_events[i].data.u32) { +#ifdef HG_HAS_SELF_FORWARD + case HG_CORE_POLL_LOOPBACK: + HG_LOG_DEBUG("HG_CORE_POLL_LOOPBACK event"); + ret = hg_core_progress_loopback_notify(context); + if (ret != HG_TIMEOUT) + HG_CHECK_HG_ERROR(done, ret, + "hg_core_progress_loopback_notify() failed"); + break; +#endif +#ifdef HG_HAS_SM_ROUTING + case HG_CORE_POLL_SM: + HG_LOG_DEBUG("HG_CORE_POLL_SM event"); + ret = hg_core_progress_na( + HG_CORE_CONTEXT_CLASS( + context)->core_class.na_sm_class, + context->core_context.na_sm_context, 0); + if (ret != HG_TIMEOUT) + HG_CHECK_HG_ERROR(done, ret, + "hg_core_progress_na() failed"); + break; +#endif + case HG_CORE_POLL_NA: + HG_LOG_DEBUG("HG_CORE_POLL_NA event"); + ret = hg_core_progress_na( + HG_CORE_CONTEXT_CLASS(context)->core_class.na_class, + context->core_context.na_context, 0); + if (ret != HG_TIMEOUT) + HG_CHECK_HG_ERROR(done, ret, + "hg_core_progress_na() failed"); + break; + default: + HG_GOTO_ERROR(done, ret, HG_INVALID_ARG, + "Invalid type of poll event (%d)", + (int) context->poll_events[i].data.u32); + } + } + + /* We progressed, will return success */ + if (nevents > 0) { ret = HG_SUCCESS; goto done; } + } else { + hg_bool_t progressed = HG_FALSE; + unsigned int progress_timeout; +#ifdef HG_HAS_SM_ROUTING + if (context->core_context.na_sm_context) { + progress_timeout = 0; + + ret = hg_core_progress_na( + HG_CORE_CONTEXT_CLASS(context)->core_class.na_sm_class, + context->core_context.na_sm_context, progress_timeout); + if (ret == HG_SUCCESS) + progressed |= HG_TRUE; + else if (ret != HG_TIMEOUT) + HG_CHECK_HG_ERROR( + done, ret, "hg_core_progress_na() failed"); + } else { +#else + progress_timeout = + (!(HG_CORE_CONTEXT_CLASS(context)->progress_mode & NA_NO_BLOCK) + && timeout && hg_core_poll_try_wait(context)) ? + (unsigned int) (remaining * 1000.0) : 0; +#endif +#ifdef HG_HAS_SM_ROUTING + } +#endif + + ret = hg_core_progress_na( + HG_CORE_CONTEXT_CLASS(context)->core_class.na_class, + context->core_context.na_context, progress_timeout); + if (ret == HG_SUCCESS) + progressed |= HG_TRUE; + else if (ret != HG_TIMEOUT) + HG_CHECK_HG_ERROR(done, ret, "hg_core_progress_na() failed"); + + /* We progressed, return success */ + if (progressed) { + ret = HG_SUCCESS; + break; + } + } + + /* There is stuff in the queues to process */ + if (!hg_atomic_queue_is_empty(context->completion_queue) + || (hg_atomic_get32(&context->backfill_queue_count) > 0)) { + ret = HG_SUCCESS; + break; } if (timeout) { @@ -3333,9 +3259,6 @@ HG_Core_context_create_id(hg_core_class_t *hg_core_class, hg_uint8_t id) { struct hg_core_private_context *context = NULL; int na_poll_fd; -#ifdef HG_HAS_SELF_FORWARD - int fd; -#endif HG_CHECK_ERROR_NORET(hg_core_class == NULL, error, "NULL HG core class"); @@ -3384,59 +3307,55 @@ HG_Core_context_create_id(hg_core_class_t *hg_core_class, hg_uint8_t id) } #endif - /* Create poll set */ - context->poll_set = hg_poll_create(); - HG_CHECK_ERROR_NORET(context->poll_set == NULL, error, - "Could not create poll set"); + /* If NA plugin exposes fd, we will use poll set and use appropriate + * progress function */ + na_poll_fd = NA_Poll_get_fd(hg_core_class->na_class, + context->core_context.na_context); -#ifdef HG_HAS_SELF_FORWARD - /* Create event for completion queue notification */ - fd = hg_event_create(); - HG_CHECK_ERROR_NORET(fd < 0, error, "Could not create event"); - context->completion_queue_notify = fd; - - /* Add event to context poll set */ - hg_poll_add(context->poll_set, fd, HG_POLLIN, - hg_core_completion_queue_notify_cb, context); -#endif + if (!HG_CORE_CONTEXT_CLASS(context)->progress_mode & NA_NO_BLOCK + && (na_poll_fd > 0)) { + struct hg_poll_event event = {.events = HG_POLLIN, .data.u64 = 0}; + int rc; - if (HG_CORE_CONTEXT_CLASS(context)->progress_mode & NA_NO_BLOCK) - /* Force to use progress poll */ - na_poll_fd = 0; - else - /* If NA plugin exposes fd, add it to poll set and use appropriate - * progress function */ - na_poll_fd = NA_Poll_get_fd(hg_core_class->na_class, - context->core_context.na_context); - if (na_poll_fd >= 0) { - hg_poll_add(context->poll_set, na_poll_fd, HG_POLLIN, - hg_core_progress_na_cb, context); - hg_poll_set_try_wait(context->poll_set, hg_core_poll_try_wait_cb, - context); - context->progress = hg_core_progress_poll; - } else - context->progress = hg_core_progress_na; + /* Create poll set */ + context->poll_set = hg_poll_create(); + HG_CHECK_ERROR_NORET(context->poll_set == NULL, error, + "Could not create poll set"); + + event.data.u32 = (hg_util_uint32_t) HG_CORE_POLL_NA; + rc = hg_poll_add(context->poll_set, na_poll_fd, &event); + HG_CHECK_ERROR_NORET( + rc != HG_UTIL_SUCCESS, error, "hg_poll_add() failed"); #ifdef HG_HAS_SM_ROUTING - /* Auto SM requires hg_core_progress_poll */ - if (context->core_context.na_sm_context) { - HG_CHECK_ERROR_NORET(context->progress != hg_core_progress_poll, error, - "Auto SM mode not supported with selected plugin"); - - if (HG_CORE_CONTEXT_CLASS(context)->progress_mode & NA_NO_BLOCK) - /* Force to use progress poll */ - na_poll_fd = 0; - else { + if (context->core_context.na_sm_context) { na_poll_fd = NA_Poll_get_fd(hg_core_class->na_sm_class, context->core_context.na_sm_context); HG_CHECK_ERROR_NORET(na_poll_fd < 0, error, "Could not get NA SM poll fd"); + + event.data.u32 = (hg_util_uint32_t) HG_CORE_POLL_SM; + rc = hg_poll_add(context->poll_set, na_poll_fd, &event); + HG_CHECK_ERROR_NORET( + rc != HG_UTIL_SUCCESS, error, "hg_poll_add() failed"); } - hg_poll_add(context->poll_set, na_poll_fd, HG_POLLIN, - hg_core_progress_na_sm_cb, context); - } #endif +#ifdef HG_HAS_SELF_FORWARD + /* Create event for completion queue notification */ + context->completion_queue_notify = hg_event_create(); + HG_CHECK_ERROR_NORET(context->completion_queue_notify < 0, error, + "Could not create event"); + + /* Add event to context poll set */ + event.data.u32 = (hg_util_uint32_t) HG_CORE_POLL_LOOPBACK; + rc = hg_poll_add( + context->poll_set, context->completion_queue_notify, &event); + HG_CHECK_ERROR_NORET( + rc != HG_UTIL_SUCCESS, error, "hg_poll_add() failed"); +#endif + } + /* Assign context ID */ context->core_context.id = id; @@ -3457,7 +3376,6 @@ HG_Core_context_destroy(hg_core_context_t *context) struct hg_core_private_context *private_context = (struct hg_core_private_context *) context; unsigned int actual_count; - int na_poll_fd; hg_util_int32_t n_handles; hg_bool_t empty; na_return_t na_ret; @@ -3540,29 +3458,23 @@ HG_Core_context_destroy(hg_core_context_t *context) } #endif - if (HG_CORE_CONTEXT_CLASS(private_context)->progress_mode & NA_NO_BLOCK) - /* Was forced to use progress poll */ - na_poll_fd = 0; - else + if (private_context->poll_set) { /* If NA plugin exposes fd, remove it from poll set */ - na_poll_fd = NA_Poll_get_fd(context->core_class->na_class, + int na_poll_fd = NA_Poll_get_fd(context->core_class->na_class, context->na_context); - if (na_poll_fd >= 0) { - rc = hg_poll_remove(private_context->poll_set, na_poll_fd); - HG_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, HG_NOENTRY, - "Could not remove NA poll descriptor from poll set"); + if (na_poll_fd > 0) { + rc = hg_poll_remove(private_context->poll_set, na_poll_fd); + HG_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, HG_NOENTRY, + "Could not remove NA poll descriptor from poll set"); + } } #ifdef HG_HAS_SM_ROUTING - if (context->na_sm_context) { - if (HG_CORE_CONTEXT_CLASS(private_context)->progress_mode & NA_NO_BLOCK) - /* Was forced to use progress poll */ - na_poll_fd = 0; - else - /* If NA plugin exposes fd, remove it from poll set */ - na_poll_fd = NA_Poll_get_fd(context->core_class->na_sm_class, - context->na_sm_context); - if (na_poll_fd >= 0) { + if (context->na_sm_context && private_context->poll_set) { + /* If NA plugin exposes fd, remove it from poll set */ + int na_poll_fd = NA_Poll_get_fd(context->core_class->na_sm_class, + context->na_sm_context); + if (na_poll_fd > 0) { rc = hg_poll_remove(private_context->poll_set, na_poll_fd); HG_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, HG_NOENTRY, "Could not remove NA poll descriptor from poll set"); @@ -3571,9 +3483,11 @@ HG_Core_context_destroy(hg_core_context_t *context) #endif /* Destroy poll set */ - rc = hg_poll_destroy(private_context->poll_set); - HG_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, HG_FAULT, - "Could not destroy poll set"); + if (private_context->poll_set) { + rc = hg_poll_destroy(private_context->poll_set); + HG_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, HG_FAULT, + "Could not destroy poll set"); + } /* Destroy NA context */ if (context->na_context) { @@ -4374,7 +4288,7 @@ HG_Core_progress(hg_core_context_t *context, unsigned int timeout) "NULL HG core context"); /* Make progress on the HG layer */ - ret = private_context->progress(private_context, timeout); + ret = hg_core_progress(private_context, timeout); HG_CHECK_ERROR_NORET(ret != HG_SUCCESS && ret != HG_TIMEOUT, done, "Could not make progress"); From a170b27aef1d36bdce82bbf8760392c84fe64eaf Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Fri, 19 Jun 2020 14:38:05 -0500 Subject: [PATCH 24/30] HG core: skip completion queue notifications if not needed --- src/mercury_core.c | 53 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 14 deletions(-) diff --git a/src/mercury_core.c b/src/mercury_core.c index 75d596f0..68762651 100644 --- a/src/mercury_core.c +++ b/src/mercury_core.c @@ -121,6 +121,7 @@ struct hg_core_private_context { struct hg_core_context core_context; /* Must remain as first field */ hg_thread_cond_t completion_queue_cond; /* Completion queue cond */ hg_thread_mutex_t completion_queue_mutex; /* Completion queue mutex */ + hg_thread_mutex_t completion_queue_notify_mutex; /* Notify mutex */ HG_QUEUE_HEAD(hg_completion_entry) backfill_queue; /* Backfill completion queue */ struct hg_atomic_queue *completion_queue; /* Default completion queue */ HG_LIST_HEAD(hg_core_private_handle) created_list; /* List of handles for that context */ @@ -132,6 +133,7 @@ struct hg_core_private_context { void *handle_create_arg; /* handle_create arg */ struct hg_poll_set *poll_set; /* Context poll set */ struct hg_poll_event poll_events[HG_CORE_MAX_EVENTS]; /* Context poll events */ + hg_atomic_int32_t completion_queue_must_notify; /* Notify of completion queue events */ hg_atomic_int32_t backfill_queue_count; /* Backfill queue count */ hg_atomic_int32_t trigger_waiting; /* Waiting in trigger */ hg_atomic_int32_t n_handles; /* Atomic used for number of handles */ @@ -2541,12 +2543,20 @@ hg_core_completion_add(struct hg_core_context *context, } #ifdef HG_HAS_SELF_FORWARD - /* TODO could prevent from self notifying if hg_poll_wait() not entered */ if (!(HG_CORE_CONTEXT_CLASS(private_context)->progress_mode & NA_NO_BLOCK) - && self_notify && private_context->completion_queue_notify) { - int rc = hg_event_set(private_context->completion_queue_notify); - HG_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, HG_FAULT, - "Could not signal completion queue"); + && self_notify && (private_context->completion_queue_notify > 0)) { + hg_thread_mutex_lock( + &private_context->completion_queue_notify_mutex); + /* Do not bother notifying if it's not needed as any event call will + * increase latency */ + if (hg_atomic_get32( + &private_context->completion_queue_must_notify)) { + int rc = hg_event_set(private_context->completion_queue_notify); + HG_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, HG_FAULT, + "Could not signal completion queue"); + } + hg_thread_mutex_unlock( + &private_context->completion_queue_notify_mutex); } #else (void) self_notify; @@ -2763,7 +2773,7 @@ static HG_INLINE hg_return_t hg_core_progress_loopback_notify(struct hg_core_private_context *context) { hg_util_bool_t progressed = HG_UTIL_FALSE; - hg_return_t ret = HG_SUCCESS; + hg_return_t ret = HG_AGAIN; int rc; rc = hg_event_get(context->completion_queue_notify, &progressed); @@ -2812,19 +2822,32 @@ hg_core_progress(struct hg_core_private_context *context, do { hg_time_t t1, t2; + hg_bool_t safe_wait = HG_FALSE; if (timeout) hg_time_get_current(&t1); - /* Only enter blocking wait if it is safe to */ if (!(HG_CORE_CONTEXT_CLASS(context)->progress_mode & NA_NO_BLOCK) - && context->poll_set && timeout && hg_core_poll_try_wait(context)) { + && timeout) { + hg_thread_mutex_lock(&context->completion_queue_notify_mutex); + + if (hg_core_poll_try_wait(context)) { + safe_wait = HG_TRUE; + hg_atomic_set32(&context->completion_queue_must_notify, 1); + } + + hg_thread_mutex_unlock(&context->completion_queue_notify_mutex); + } + + /* Only enter blocking wait if it is safe to */ + if (context->poll_set && safe_wait) { unsigned int i, nevents; int rc; rc = hg_poll_wait(context->poll_set, (unsigned int) (remaining * 1000.0), HG_CORE_MAX_EVENTS, context->poll_events, &nevents); + hg_atomic_set32(&context->completion_queue_must_notify, 0); HG_CHECK_ERROR(rc != HG_UTIL_SUCCESS, done, ret, HG_PROTOCOL_ERROR, "hg_poll_wait() failed"); @@ -2834,9 +2857,8 @@ hg_core_progress(struct hg_core_private_context *context, case HG_CORE_POLL_LOOPBACK: HG_LOG_DEBUG("HG_CORE_POLL_LOOPBACK event"); ret = hg_core_progress_loopback_notify(context); - if (ret != HG_TIMEOUT) - HG_CHECK_HG_ERROR(done, ret, - "hg_core_progress_loopback_notify() failed"); + HG_CHECK_HG_ERROR(done, ret, + "hg_core_progress_loopback_notify() failed"); break; #endif #ifdef HG_HAS_SM_ROUTING @@ -2890,9 +2912,7 @@ hg_core_progress(struct hg_core_private_context *context, } else { #else progress_timeout = - (!(HG_CORE_CONTEXT_CLASS(context)->progress_mode & NA_NO_BLOCK) - && timeout && hg_core_poll_try_wait(context)) ? - (unsigned int) (remaining * 1000.0) : 0; + safe_wait ? (unsigned int) (remaining * 1000.0) : 0; #endif #ifdef HG_HAS_SM_ROUTING } @@ -3285,6 +3305,10 @@ HG_Core_context_create_id(hg_core_class_t *hg_core_class, hg_uint8_t id) /* No handle created yet */ hg_atomic_init32(&context->n_handles, 0); + /* Notifications of completion queue events */ + hg_atomic_init32(&context->completion_queue_must_notify, 0); + hg_thread_mutex_init(&context->completion_queue_notify_mutex); + /* Initialize completion queue mutex/cond */ hg_thread_mutex_init(&context->completion_queue_mutex); hg_thread_cond_init(&context->completion_queue_cond); @@ -3512,6 +3536,7 @@ HG_Core_context_destroy(hg_core_context_t *context) context->data_free_callback(context->data); /* Destroy completion queue mutex/cond */ + hg_thread_mutex_destroy(&private_context->completion_queue_notify_mutex); hg_thread_mutex_destroy(&private_context->completion_queue_mutex); hg_thread_cond_destroy(&private_context->completion_queue_cond); hg_thread_spin_destroy(&private_context->pending_list_lock); From c45659ae0425d484f1a555d4907ea5fdde8bf26e Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Fri, 19 Jun 2020 17:04:34 -0500 Subject: [PATCH 25/30] HG core: fix busy spin accidentally inserted into previous commit --- src/mercury_core.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/mercury_core.c b/src/mercury_core.c index 68762651..1840be7a 100644 --- a/src/mercury_core.c +++ b/src/mercury_core.c @@ -2792,7 +2792,6 @@ hg_core_progress_loopback_notify(struct hg_core_private_context *context) static HG_INLINE hg_bool_t hg_core_poll_try_wait(struct hg_core_private_context *context) { - return HG_FALSE; /* Something is in one of the completion queues */ if (!hg_atomic_queue_is_empty(context->completion_queue) || (hg_atomic_get32(&context->backfill_queue_count) > 0)) From 663453b7258b4f055f4093617ff094bc8be18b6d Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Mon, 22 Jun 2020 15:13:37 -0500 Subject: [PATCH 26/30] HG util: fix atomic_long type on MacOS --- src/util/mercury_atomic.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/mercury_atomic.h b/src/util/mercury_atomic.h index 7a684b98..48757225 100644 --- a/src/util/mercury_atomic.h +++ b/src/util/mercury_atomic.h @@ -33,7 +33,7 @@ typedef OPA_ptr_t hg_atomic_int64_t; /* OPA has only limited 64-bit support */ #elif defined(HG_UTIL_HAS_STDATOMIC_H) # include typedef atomic_int hg_atomic_int32_t; -# if HG_UTIL_ATOMIC_LONG_WIDTH == 8 +# if (HG_UTIL_ATOMIC_LONG_WIDTH == 8) && !defined(__APPLE__) typedef atomic_long hg_atomic_int64_t; # else typedef atomic_llong hg_atomic_int64_t; From 9a08a8e8a8af557563594936e12eb46fb02a2ead Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Mon, 22 Jun 2020 15:14:44 -0500 Subject: [PATCH 27/30] HG Core: fix missing parentheses --- src/mercury_core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mercury_core.c b/src/mercury_core.c index 1840be7a..f11e53c6 100644 --- a/src/mercury_core.c +++ b/src/mercury_core.c @@ -3335,7 +3335,7 @@ HG_Core_context_create_id(hg_core_class_t *hg_core_class, hg_uint8_t id) na_poll_fd = NA_Poll_get_fd(hg_core_class->na_class, context->core_context.na_context); - if (!HG_CORE_CONTEXT_CLASS(context)->progress_mode & NA_NO_BLOCK + if (!(HG_CORE_CONTEXT_CLASS(context)->progress_mode & NA_NO_BLOCK) && (na_poll_fd > 0)) { struct hg_poll_event event = {.events = HG_POLLIN, .data.u64 = 0}; int rc; From ec99dd793b268cdeae02bb90ff72937a6f7ba856 Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Mon, 22 Jun 2020 16:21:23 -0500 Subject: [PATCH 28/30] NA SM: add CMA documentation when encountering EPERM error (fix #344) --- src/na/na_sm.c | 67 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 63 insertions(+), 4 deletions(-) diff --git a/src/na/na_sm.c b/src/na/na_sm.c index fdaedb11..5efbc337 100644 --- a/src/na/na_sm.c +++ b/src/na/na_sm.c @@ -355,6 +355,12 @@ struct na_sm_class { static char * getlogin_safe(void); +/** + * Get value from ptrace_scope. + */ +static int +na_sm_get_ptrace_scope_value(void); + /** * Convert errno to NA return values. */ @@ -1091,6 +1097,29 @@ getlogin_safe(void) return passwd ? passwd->pw_name : "unknown"; } +/*---------------------------------------------------------------------------*/ +static int +na_sm_get_ptrace_scope_value(void) +{ + FILE *file; + int val = 0, rc; + + /* Try to open ptrace_scope */ + file = fopen("/proc/sys/kernel/yama/ptrace_scope", "r"); + if (file) { + rc = fscanf(file, "%d", &val); + NA_CHECK_ERROR_NORET( + rc != 1, done, "Could not get value from ptrace_scope"); + + rc = fclose(file); + NA_CHECK_ERROR_NORET( + rc != 0, done, "fclose() failed (%s)", strerror(errno)); + } + +done: + return val; +} + /*---------------------------------------------------------------------------*/ static na_return_t na_sm_errno_to_na(int rc) @@ -4256,8 +4285,23 @@ na_sm_put(na_class_t *na_class, na_context_t *context, na_cb_t callback, #if defined(NA_SM_HAS_CMA) nwrite = process_vm_writev(na_sm_addr->pid, local_iov, liovcnt, remote_iov, riovcnt, /* unused */ 0); - NA_CHECK_ERROR(nwrite < 0, error, ret, na_sm_errno_to_na(errno), - "process_vm_writev() failed (%s)", strerror(errno)); + if (unlikely(nwrite < 0)) { + if ((errno == EPERM) && na_sm_get_ptrace_scope_value()) { + NA_GOTO_ERROR(error, ret, na_sm_errno_to_na(errno), + "process_vm_writev() failed (%s):\n", + "Kernel Yama configuration does not allow cross-memory attach, " + "either run as root: \n" + "# /usr/sbin/sysctl kernel.yama.ptrace_scope=0\n" + "or if set to restricted, add the following call to your " + "application:\n" + "prctl(PR_SET_PTRACER, PR_SET_PTRACER_ANY, 0, 0, 0);\n" + "See https://www.kernel.org/doc/Documentation/security/Yama.txt" + " for more details.", + strerror(errno)); + } else + NA_GOTO_ERROR(error, ret, na_sm_errno_to_na(errno), + "process_vm_writev() failed (%s)", strerror(errno)); + } NA_CHECK_ERROR((na_size_t) nwrite != length, error, ret, NA_MSGSIZE, "Wrote %ld bytes, was expecting %lu bytes", nwrite, length); #elif defined(__APPLE__) @@ -4378,8 +4422,23 @@ na_sm_get(na_class_t *na_class, na_context_t *context, na_cb_t callback, #if defined(NA_SM_HAS_CMA) nread = process_vm_readv(na_sm_addr->pid, local_iov, liovcnt, remote_iov, riovcnt, /* unused */ 0); - NA_CHECK_ERROR(nread < 0, error, ret, na_sm_errno_to_na(errno), - "process_vm_readv() failed (%s)", strerror(errno)); + if (unlikely(nread < 0)) { + if ((errno == EPERM) && na_sm_get_ptrace_scope_value()) { + NA_GOTO_ERROR(error, ret, na_sm_errno_to_na(errno), + "process_vm_readv() failed (%s):\n", + "Kernel Yama configuration does not allow cross-memory attach, " + "either run as root: \n" + "# /usr/sbin/sysctl kernel.yama.ptrace_scope=0\n" + "or if set to restricted, add the following call to your " + "application:\n" + "prctl(PR_SET_PTRACER, PR_SET_PTRACER_ANY, 0, 0, 0);\n" + "See https://www.kernel.org/doc/Documentation/security/Yama.txt" + " for more details.", + strerror(errno)); + } else + NA_CHECK_ERROR(nread < 0, error, ret, na_sm_errno_to_na(errno), + "process_vm_readv() failed (%s)", strerror(errno)); + } #elif defined(__APPLE__) kret = task_for_pid(mach_task_self(), na_sm_addr->pid, &remote_task); NA_CHECK_ERROR(kret != KERN_SUCCESS, error, ret, NA_PERMISSION, From e6a0cf2cdd9de448cb6524e2ea97e8e10e21ad84 Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Mon, 22 Jun 2020 19:31:58 -0500 Subject: [PATCH 29/30] HG util: make cpu_spinwait use intrinsics --- src/util/mercury_atomic_queue.h | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/util/mercury_atomic_queue.h b/src/util/mercury_atomic_queue.h index 73c0b153..16898460 100644 --- a/src/util/mercury_atomic_queue.h +++ b/src/util/mercury_atomic_queue.h @@ -44,6 +44,21 @@ #include "mercury_atomic.h" #include "mercury_mem.h" +/* For busy loop spinning */ +#ifndef cpu_spinwait +# if defined(_WIN32) +# define cpu_spinwait YieldProcessor +# elif defined(__x86_64__) || defined(__i386__) +# include +# define cpu_spinwait _mm_pause +# elif defined(__arm__) +# define cpu_spinwait() __asm__ __volatile__("yield") +# else +# warning "Processor yield is not supported on this architecture." +# define cpu_spinwait +# endif +#endif + /*************************************/ /* Public Type and Struct Definition */ /*************************************/ @@ -66,14 +81,6 @@ struct hg_atomic_queue { /* Public Macros */ /*****************/ -#ifndef cpu_spinwait -# if defined(__x86_64__) || defined(__amd64__) -# define cpu_spinwait() asm volatile("pause\n" : : : "memory"); -# else -# define cpu_spinwait() ; -# endif -#endif - /*********************/ /* Public Prototypes */ /*********************/ From 1794ba75ab3d988951eb16140a5124b26e9ef9bf Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Tue, 23 Jun 2020 13:33:05 -0500 Subject: [PATCH 30/30] v2.0.0rc1 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 009ff110..4111d137 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -2.0.0a1 +2.0.0rc1