Merge pull request #742 from MichaelBrim/crusher-fixes

Fixes to enable running on OLCF Crusher/Frontier
LLNL · Nov 23, 2022 · a13edaf · a13edaf
2 parents 98f5a9c + c4ac035
commit a13edaf
Show file tree

Hide file tree

Showing 12 changed files with 142 additions and 57 deletions.
diff --git a/Makefile.am b/Makefile.am
@@ -5,8 +5,10 @@ CONFIG = ordered
 ACLOCAL_AMFLAGS = -I m4
 
 pkgconfigdir = @pkgconfigdir@
-pkgconfig_DATA = client/unifyfs.pc \
-		 client/unifyfs-api.pc
+pkgconfig_DATA = \
+  client/unifyfs.pc \
+  client/unifyfs-api.pc \
+  client/unifyfs-static.pc
 
 CLEANFILES =
 

diff --git a/client/src/Makefile.am b/client/src/Makefile.am
@@ -44,15 +44,13 @@ CLIENT_COMMON_CPPFLAGS = \
 
 CLIENT_COMMON_CFLAGS = \
   $(AM_CFLAGS) \
-  $(UNIFYFS_COMMON_FLAGS) \
-  $(MPI_CFLAGS)
+  $(UNIFYFS_COMMON_FLAGS)
 
 CLIENT_COMMON_LDFLAGS = \
   -version-info $(LIBUNIFYFS_LT_VERSION)
 
 CLIENT_COMMON_LIBADD = \
   $(UNIFYFS_COMMON_LIBS) \
-  $(MPI_CLDFLAGS) \
   -lm -lrt -lcrypto -lpthread
 
 if HAVE_SPATH
@@ -124,8 +122,8 @@ libunifyfs_la_SOURCES  = \
 
 if USE_PMPI_WRAPPERS
 libunifyfs_mpi_la_CPPFLAGS = $(CLIENT_COMMON_CPPFLAGS)
-libunifyfs_mpi_la_CFLAGS   = $(CLIENT_COMMON_CFLAGS)
-libunifyfs_mpi_la_LDFLAGS  = $(CLIENT_COMMON_LDFLAGS)
+libunifyfs_mpi_la_CFLAGS   = $(CLIENT_COMMON_CFLAGS) $(MPI_CFLAGS)
+libunifyfs_mpi_la_LDFLAGS  = $(CLIENT_COMMON_LDFLAGS) $(MPI_CLDFLAGS)
 libunifyfs_mpi_la_LIBADD   = libunifyfs.la
 libunifyfs_mpi_la_SOURCES  = $(PMPI_SRC_FILES)
 endif #USE_PMPI_WRAPPERS
@@ -153,8 +151,8 @@ endif #ENABLE_LD_PRELOAD
 
 if USE_PMPI_WRAPPERS
 libunifyfs_mpi_gotcha_la_CPPFLAGS = $(CLIENT_COMMON_CPPFLAGS)
-libunifyfs_mpi_gotcha_la_CFLAGS   = $(CLIENT_COMMON_CFLAGS)
-libunifyfs_mpi_gotcha_la_LDFLAGS  = $(CLIENT_COMMON_LDFLAGS)
+libunifyfs_mpi_gotcha_la_CFLAGS   = $(CLIENT_COMMON_CFLAGS) $(MPI_CFLAGS)
+libunifyfs_mpi_gotcha_la_LDFLAGS  = $(CLIENT_COMMON_LDFLAGS) $(MPI_CLDFLAGS)
 libunifyfs_mpi_gotcha_la_LIBADD   = libunifyfs_gotcha.la
 libunifyfs_mpi_gotcha_la_SOURCES  = $(PMPI_SRC_FILES)
 endif #USE_PMPI_WRAPPERS

diff --git a/client/unifyfs-api.pc.in b/client/unifyfs-api.pc.in
@@ -4,9 +4,8 @@ libdir=@libdir@
 includedir=@includedir@
 
 Name: UnifyFS
-Description: client library for UnifyFS unified burst buffer file system
+Description: API library for UnifyFS unified local storage file system
 Version: @LIBUNIFYFS_API_VERSION@
-Requires:
+Requires.private: margo
 Libs: -L${libdir} -lunifyfs_api
-Cflags: -I${includedir}
-
+Cflags: -I${includedir} -I${includedir}/unifyfs
diff --git a/client/unifyfs-static.pc.in b/client/unifyfs-static.pc.in
@@ -0,0 +1,12 @@
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: UnifyFS
+Description: client library for UnifyFS unified local storage file system
+Version: @LIBUNIFYFS_API_VERSION@
+Requires: margo
+Libs: @LINK_WRAPPERS@ ${libdir}/libunifyfs.a @SPATH_LIBS@ -lcrypto -lm -lrt -lpthread -lz
+Cflags: -I${includedir} -I${includedir}/unifyfs -D__FILE_OFFSET_BITS=64
+
diff --git a/client/unifyfs.pc.in b/client/unifyfs.pc.in
@@ -4,9 +4,8 @@ libdir=@libdir@
 includedir=@includedir@
 
 Name: UnifyFS
-Description: client library for UnifyFS unified burst buffer file system
+Description: client library for UnifyFS unified local storage file system
 Version: @LIBUNIFYFS_API_VERSION@
-Requires:
+Requires.private: margo
 Libs: -L${libdir} -lunifyfs_gotcha
-Cflags: -I${includedir} -D__FILE_OFFSET_BITS=64
-
+Cflags: -I${includedir} -I${includedir}/unifyfs -D__FILE_OFFSET_BITS=64
diff --git a/configure.ac b/configure.ac
@@ -377,8 +377,10 @@ AC_CONFIG_FILES([Makefile
                  client/src/Makefile
                  client/unifyfs.pc
                  client/unifyfs-api.pc
+                 client/unifyfs-static.pc
                  examples/Makefile
                  examples/src/Makefile
+                 examples/src/Makefile.examples
                  extras/Makefile
                  extras/unifyfs.conf
                  t/Makefile

diff --git a/examples/src/Makefile.am b/examples/src/Makefile.am
@@ -1,3 +1,7 @@
+makefiledir = $(datadir)/makefiles
+
+makefile_DATA = Makefile.examples
+
 testutil_headers = \
   testutil.h \
   testutil_rdwr.h
@@ -61,8 +65,9 @@ endif #HAVE_FORTRAN
 
 endif #HAVE_GOTCHA
 
-CLEANFILES = $(libexec_PROGRAMS)
+CLEANFILES = Makefile.examples $(libexec_PROGRAMS)
 
+EXTRA_DIST = Makefile.examples.in
 
 # Common compile/link flag definitions
 

diff --git a/examples/src/Makefile.examples.in b/examples/src/Makefile.examples.in
@@ -0,0 +1,62 @@
+UNIFYFS_INSTALL = @prefix@
+
+MPICC ?= mpicc
+
+UNIFYFS_CFG    = $(UNIFYFS_INSTALL)/bin/unifyfs-config
+UNIFYFS_PKGCFG = PKG_CONFIG_PATH=$(PKG_CONFIG_PATH):$(UNIFYFS_INSTALL)/lib/pkgconfig pkg-config
+
+GOTCHA_PROGRAMS = \
+  app-btio-gotcha \
+  app-mpiio-gotcha \
+  app-tileio-gotcha \
+  checkpoint-restart-gotcha \
+  multi-write-gotcha \
+  read-gotcha \
+  read-data-gotcha \
+  simul-gotcha \
+  transfer-gotcha \
+  write-gotcha \
+  writeread-gotcha \
+  write-transfer-gotcha
+
+STATIC_PROGRAMS += \
+  app-btio-static \
+  app-mpiio-static \
+  app-tileio-static \
+  checkpoint-restart-static \
+  multi-write-static \
+  read-static \
+  read-data-static \
+  simul-static \
+  transfer-static \
+  write-static \
+  writeread-static \
+  write-transfer-static
+
+LIBS += -lm -lrt
+
+.PHONY: default all clean gotcha static
+
+default: all
+
+all: gotcha static
+
+clean:
+	$(RM) *.o
+	$(RM) $(GOTCHA_PROGRAMS)
+	$(RM) $(STATIC_PROGRAMS)
+
+gotcha: $(GOTCHA_PROGRAMS)
+
+static: $(STATIC_PROGRAMS)
+
+testutil.o: testutil.c
+	$(MPICC) $(CPPFLAGS) $(CFLAGS) `$(UNIFYFS_PKGCFG) --cflags unifyfs-api` -c -o testutil.o testutil.c
+
+%-gotcha: %.c testutil.o
+	$(MPICC) $(CPPFLAGS) $(CFLAGS) `$(UNIFYFS_PKGCFG) --cflags unifyfs-api` -c -o $<.o $<
+	$(MPICC) -o $@ $<.o testutil.o $(LDFLAGS) `$(UNIFYFS_PKGCFG) --libs unifyfs` $(LIBS)
+
+%-static: %.c testutil.o
+	$(MPICC) $(CPPFLAGS) $(CFLAGS) `$(UNIFYFS_PKGCFG) --cflags unifyfs-static` -c -o $<.o $<
+	$(MPICC) $(LDFLAGS) `$(UNIFYFS_PKGCFG) --libs-only-L unifyfs-static` -o $@ $<.o testutil.o `$(UNIFYFS_PKGCFG) --libs-only-other --libs-only-l unifyfs-static` $(LIBS)
diff --git a/examples/src/app-mpiio.c b/examples/src/app-mpiio.c
@@ -12,20 +12,16 @@
  * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text.
  */
 
-#include <config.h>
 
+#include <getopt.h>
+#include <libgen.h>
+#include <limits.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <stdint.h>
 #include <string.h>
-#include <errno.h>
-#include <limits.h>
-#include <sys/time.h>
-#include <sys/types.h>
 #include <unistd.h>
-#include <fcntl.h>
-#include <libgen.h>
-#include <getopt.h>
+
 #include <mpi.h>
 #include <unifyfs.h>
 
@@ -190,8 +186,6 @@ static void report_result(void)
 
     min_io_bw = (per_rank_mib * total_ranks) / max_io_time;
 
-    errno = 0;
-
     test_print_once(rank,
                     "\n"
                     "Number of processes:     %d\n"

diff --git a/examples/src/read-data.c b/examples/src/read-data.c
@@ -17,22 +17,18 @@
  * specifying filename) or non-interactively (specifying filename with offset
  * and length).
  */
-#include <config.h>
 
+#include <errno.h>
+#include <getopt.h>
+#include <libgen.h>
+#include <limits.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <stdint.h>
 #include <string.h>
-#include <errno.h>
-#include <limits.h>
-#include <sys/time.h>
-#include <sys/types.h>
+#include <time.h>
 #include <unistd.h>
-#include <fcntl.h>
-#include <libgen.h>
-#include <getopt.h>
+
 #include <unifyfs.h>
-#include <time.h>
 
 #include "testutil.h"
 
@@ -115,18 +111,19 @@ static void random_offlen(uint64_t filesize, uint64_t maxoff, uint64_t maxlen,
 
 static void do_pread(int fd, size_t length, off_t offset)
 {
+    int err;
     ssize_t ret = 0;
     struct timespec ts1, ts2;
     double ts1nsec, ts2nsec;
     double elapsed_sec, mbps;
 
     alloc_buf(length);
 
-    errno = 0;
-
     clock_gettime(CLOCK_REALTIME, &ts1);
 
+    errno = 0;
     ret = pread(fd, buf, length, offset);
+    err = errno;
 
     clock_gettime(CLOCK_REALTIME, &ts2);
 
@@ -137,8 +134,8 @@ static void do_pread(int fd, size_t length, off_t offset)
     mbps = (1.0 * length / (1<<20)) / elapsed_sec;
 
     printf(" -> pread(off=%lu, len=%lu) = %zd", offset, length, ret);
-    if (errno) {
-        printf("  (err=%d, %s)\n", errno, strerror(errno));
+    if (err) {
+        printf("  (err=%d, %s)\n", err, strerror(err));
     } else {
         printf(" (%.3f sec, %.3lf MB/s)\n", elapsed_sec, mbps);
 

diff --git a/examples/src/transfer.c b/examples/src/transfer.c
@@ -12,23 +12,15 @@
  * Please read https://github.com/LLNL/UnifyFS/LICENSE for full license text.
  */
 
-#include <config.h>
 
+#include <getopt.h>
+#include <libgen.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <stdint.h>
 #include <string.h>
-#include <errno.h>
-#include <limits.h>
-#include <sys/time.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <dirent.h>
-#include <libgen.h>
-#include <getopt.h>
 #include <time.h>
+
+
 #include <mpi.h>
 #include <unifyfs.h>
 
@@ -84,7 +76,6 @@ int main(int argc, char** argv)
     int ret = 0;
     int ch = 0;
     int optidx = 0;
-    struct stat sb = { 0, };
 
     size_t srclen;
     char* srcpath;

diff --git a/util/unifyfs/src/unifyfs-rm.c b/util/unifyfs/src/unifyfs-rm.c
@@ -803,7 +803,7 @@ static int generic_stage(char* cmd, int run_argc, unifyfs_args_t* args)
     construct_stage_argv(args, argv + run_argc);
     if (args->debug) {
         for (int i = 0; i < (argc - 1); i++) {
-            fprintf(stdout, "UNIFYFS DEBUG: stage_argv[%d] = %s\n",
+            fprintf(stdout, "UNIFYFS STAGE DEBUG: stage_argv[%d] = %s\n",
                     i, argv[i]);
             fflush(stdout);
         }
@@ -854,6 +854,14 @@ static int jsrun_launch(unifyfs_resource_t* resource,
     argv[12] = strdup("-a1");
     construct_server_argv(args, argv + jsrun_argc);
 
+    if (args->debug) {
+        for (int i = 0; i < (argc - 1); i++) {
+            fprintf(stdout, "UNIFYFS LAUNCH DEBUG: jsrun argv[%d] = %s\n",
+                    i, argv[i]);
+            fflush(stdout);
+        }
+    }
+
     execvp(argv[0], argv);
     perror("failed to execvp() jsrun to launch unifyfsd");
     return -errno;
@@ -956,6 +964,14 @@ static int mpirun_launch(unifyfs_resource_t* resource,
     argv[4] = strdup("ppr:1:node");
     construct_server_argv(args, argv + mpirun_argc);
 
+    if (args->debug) {
+        for (int i = 0; i < (argc - 1); i++) {
+            fprintf(stdout, "UNIFYFS LAUNCH DEBUG: mpirun argv[%d] = %s\n",
+                    i, argv[i]);
+            fflush(stdout);
+        }
+    }
+
     execvp(argv[0], argv);
     perror("failed to execvp() mpirun to launch unifyfsd");
     return -errno;
@@ -1049,12 +1065,20 @@ static int srun_launch(unifyfs_resource_t* resource,
     argv = calloc(argc, sizeof(char*));
     argv[0] = strdup("srun");
     argv[1] = strdup("--exact");
-    argv[2] = strdup("--overcommit");
+    argv[2] = strdup("--overlap");
     argv[3] = strdup(n_nodes);
     argv[4] = strdup("--ntasks-per-node=1");
     argv[5] = strdup(n_cores);
     construct_server_argv(args, argv + srun_argc);
 
+    if (args->debug) {
+        for (int i = 0; i < (argc - 1); i++) {
+            fprintf(stdout, "UNIFYFS LAUNCH DEBUG: srun argv[%d] = %s\n",
+                    i, argv[i]);
+            fflush(stdout);
+        }
+    }
+
     execvp(argv[0], argv);
     perror("failed to execvp() srun to launch unifyfsd");
     return -errno;