diff --git a/autotools/buildme b/autotools/buildme new file mode 100755 index 00000000000..5d5533f8850 --- /dev/null +++ b/autotools/buildme @@ -0,0 +1,56 @@ +#!/bin/bash + +set -x + +installdir=`pwd`/install + +# add autotools install bin to our path +export PATH=${installdir}/bin:$PATH + +# build autoconf +if [ ! -f autoconf-2.69.tar.gz ] ; then + wget http://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz +fi +rm -rf autoconf-2.69 +tar -zxf autoconf-2.69.tar.gz +pushd autoconf-2.69 + ./configure --prefix=$installdir + make + make install +popd + +# build automake +if [ ! -f automake-1.15.tar.gz ] ; then + wget http://ftp.gnu.org/gnu/automake/automake-1.15.tar.gz +fi +rm -rf automake-1.15 +tar -zxf automake-1.15.tar.gz +pushd automake-1.15 + ./configure --prefix=$installdir + make + make install +popd + +# build libtool +if [ ! -f libtool-2.4.6.tar.gz ] ; then + wget http://mirror.team-cymru.org/gnu/libtool/libtool-2.4.6.tar.gz +fi +rm -rf libtool-2.4.6 +tar -zxf libtool-2.4.6.tar.gz +pushd libtool-2.4.6 + ./configure --prefix=$installdir + make + make install +popd + +# build pkg-config +if [ ! -f pkg-config-0.27.1.tar.gz ] ; then + wget https://pkg-config.freedesktop.org/releases/pkg-config-0.27.1.tar.gz +fi +rm -rf pkg-config-0.27.1 +tar -zxf pkg-config-0.27.1.tar.gz +pushd pkg-config-0.27.1 + ./configure --prefix=$installdir + make + make install +popd diff --git a/buildme b/buildme new file mode 100755 index 00000000000..366c7bf1918 --- /dev/null +++ b/buildme @@ -0,0 +1,122 @@ +#!/bin/bash + +# run this to apply patches and create a new mv2 tarball + +set -x + +export PKG_CONFIG_PATH="/path/to/unifyfs/install/lib/pkgconfig":$PKG_CONFIG_PATH + +# build autotools that we need (one of the patches below changes configure) +pushd autotools + ./buildme +popd + +# add autotools install bin to our path +export PATH=`pwd`/autotools/install/bin:$PATH + +# MPI installation directory +installdir=`pwd`/install-gnu-main +rm -rf $installdir + +# GNU compiler settings +export CC=gcc +export CXX=g++ +export F77=gfortran +export FC=gfortran +#export MPICHLIB_CFLAGS="-g -O2" +#export MPICHLIB_CXXFLAGS="-g -O2" +#export MPICHLIB_FFLAGS="-g -O2 -fno-second-underscore" +#export MPICHLIB_FCFLAGS="-g -O2 -fno-second-underscore" +export CFLAGS="-I/usr/include/slurm" +export MPICHLIB_CFLAGS="-g -O0 -I/usr/include/slurm" +export MPICHLIB_CXXFLAGS="-g -O0" +export MPICHLIB_FFLAGS="-g -O0 -fno-second-underscore" +export MPICHLIB_FCFLAGS="-g -O0 -fno-second-underscore" + +# Intel compiler settings +#export CC=icc +#export CXX=icpc +#export F77=ifort +#export FC=ifort +#export MPICHLIB_CFLAGS="-g -O2" +#export MPICHLIB_CXXFLAGS="-g -O2" +#export MPICHLIB_FFLAGS="-g -O2" +#export MPICHLIB_FCFLAGS="-g -O2" +#export MPICHLIB_CFLAGS="-g -O0" +#export MPICHLIB_CXXFLAGS="-g -O0" +#export MPICHLIB_FFLAGS="-g -O0" +#export MPICHLIB_FCFLAGS="-g -O0" + +# PGI compiler settings +#export CC=pgcc +#export CXX=pgCC +#export F77=pgf77 +##export FC=pgf90 +#export MPICHLIB_CFLAGS="-g -O2 -fPIC" +#export MPICHLIB_CXXFLAGS="-g -O2 -fPIC" +#export MPICHLIB_FFLAGS="-g -O2 -fPIC" +#export MPICHLIB_FCFLAGS="-g -O2 -fPIC" + +# Pathscale compiler settings +#export CC=pathcc +#export CXX=pathCC +#export F77=pathf90 +##export FC=pathf90 +#export MPICHLIB_CFLAGS="-g -O2" +#export MPICHLIB_CXXFLAGS="-g -O2" +#export MPICHLIB_FFLAGS="-g -O2 -fno-second-underscore" +#export MPICHLIB_FCFLAGS="-g -O2 -fno-second-underscore" + +#export CONFIG_FLAGS="--disable-fast --enable-g=dbg --enable-error-checking=runtime --enable-error-messages=all --enable-nmpi-as-mpi" +export CONFIG_FLAGS="--enable-fast=all --enable-g=dbg --enable-nmpi-as-mpi" + +# PMI library: PMGR_COLLECTIVE, SLURM +export PMI="--with-pm=no --with-pmi=slurm" +#export PMI="--with-pm=slurm --with-pmi=pmi2" +#export PMI="--with-pm=no --with-pmi=pmi2" + +# Network devices: PSM, Mellanox, Shared Memory, Mellanox + CUDA +#export DEVICE_FLAGS="--with-device=ch3:psm --with-psm2" +#export DEVICE_FLAGS="--with-device=ch3:psm --with-psm2 --with-psm2-lib=/usr/lib64" +#export DEVICE_FLAGS="--with-device=ch3:psm --with-psm" +#export DEVICE_FLAGS="--with-device=ch3:psm --with-psm --with-psm-lib=/usr/lib64" +#export DEVICE_FLAGS="--with-device=ch3:psm" +#export DEVICE_FLAGS="--with-device=ch3:mrail --with-rdma=gen2" +#export DEVICE_FLAGS="--with-device=ch3:mrail --with-rdma=gen2 --enable-cuda --with-cuda=/opt/cudatoolkit-5.0" +#export DEVICE_FLAGS="--with-device=ch3:nemesis" +export DEVICE_FLAGS="--with-device=ch3" + +#export REG_CACHE="--disable-registration-cache" +export REG_CACHE="" + +./autogen.sh + +rm -rf build +mkdir build +pushd build + +../configure \ + --prefix=${installdir} \ + --enable-f77 --enable-fc --enable-cxx \ + ${CONFIG_FLAGS} \ + --enable-shared --enable-sharedlibs=gcc \ + --enable-debuginfo \ + --disable-new-dtags \ + ${PMI} \ + ${DEVICE_FLAGS} \ + ${REG_CACHE} \ + --with-ch3-rank-bits=32 \ + --enable-romio --with-file-system=lustre+nfs+ufs \ + --disable-mpe --without-mpe 2>&1 > configure.out + +# --enable-romio --with-file-system=gpfs+nfs+ufs \ +# --enable-romio --with-file-system=lustre+nfs+ufs \ +#../configure \ +#--build=x86_64-redhat-linux-gnu --host=x86_64-redhat-linux-gnu --program-prefix= --disable-dependency-tracking --prefix=$installdir/mvapich2-x/gnu4.9.3/ifs/basic/slurm --exec-prefix=${installdir}/mvapich2-x/gnu4.9.3/ifs/basic/slurm --bindir=${installdir}/mvapich2-x/gnu4.9.3/ifs/basic/slurm/bin --sbindir=${installdir}/mvapich2-x/gnu4.9.3/ifs/basic/slurm/sbin --sysconfdir=${installdir}/mvapich2-x/gnu4.9.3/ifs/basic/slurm/etc --datadir=${installdir}/mvapich2-x/gnu4.9.3/ifs/basic/slurm/share --includedir=${installdir}/mvapich2-x/gnu4.9.3/ifs/basic/slurm/include --libdir=${installdir}/mvapich2-x/gnu4.9.3/ifs/basic/slurm/lib64 --libexecdir=${installdir}/mvapich2-x/gnu4.9.3/ifs/basic/slurm/libexec --localstatedir=/var --sharedstatedir=/var/lib --mandir=${installdir}/mvapich2-x/gnu4.9.3/ifs/basic/slurm/share/man --infodir=${installdir}/mvapich2-x/gnu4.9.3/ifs/basic/slurm/share/info CC=gcc CXX=g++ F77=gfortran FC=gfortran --enable-ucr --disable-rpath --disable-static --enable-shared --disable-rdma-cm --without-hydra-ckpointlib --with-pm=slurm --with-pmi=pmi2 --with-device=ch3:psm --disable-mcast --enable-fast=O3 --enable-f77 -enable-fc --enable-cxx --enable-nmpi-as-mpi --enable-shared --enable-sharedlibs=gcc --disable-new-dtags --disable-registration-cache --with-ch3-rank-bits=32 --enable-romio --with-file-system=lustre+nfs+ufs --disable-mpe --without-mpe LDFLAGS=-Wl,-rpath,XORIGIN/placeholder 2>&1 > configure.out +#exit 0 + +#make clean + +#make -j8 VERBOSE=1 2>&1 > make.out + +make install VERBOSE=1 diff --git a/src/mpi/romio/adio/ad_gpfs/ad_gpfs.c b/src/mpi/romio/adio/ad_gpfs/ad_gpfs.c index 203fbdedeff..1ff59f5c961 100644 --- a/src/mpi/romio/adio/ad_gpfs/ad_gpfs.c +++ b/src/mpi/romio/adio/ad_gpfs/ad_gpfs.c @@ -45,6 +45,7 @@ struct ADIOI_Fns_struct ADIO_GPFS_operations = { ADIOI_GEN_IreadStrided, /* IreadStrided */ ADIOI_GEN_IwriteStrided, /* IwriteStrided */ ADIOI_GPFS_Flush, /* Flush */ + ADIOI_GEN_Fence, /* Fence */ ADIOI_GEN_Resize, /* Resize */ ADIOI_GEN_Delete, /* Delete */ ADIOI_GEN_Feature, /* Features */ diff --git a/src/mpi/romio/adio/ad_lustre/ad_lustre.c b/src/mpi/romio/adio/ad_lustre/ad_lustre.c index b7f5108bbf2..50581d736c1 100644 --- a/src/mpi/romio/adio/ad_lustre/ad_lustre.c +++ b/src/mpi/romio/adio/ad_lustre/ad_lustre.c @@ -32,6 +32,7 @@ struct ADIOI_Fns_struct ADIO_LUSTRE_operations = { ADIOI_GEN_IreadStrided, /* IreadStrided */ ADIOI_GEN_IwriteStrided, /* IwriteStrided */ ADIOI_GEN_Flush, /* Flush */ + ADIOI_GEN_Fence, /* Fence */ ADIOI_GEN_Resize, /* Resize */ ADIOI_GEN_Delete, /* Delete */ ADIOI_GEN_Feature, /* Features */ diff --git a/src/mpi/romio/adio/ad_nfs/ad_nfs.c b/src/mpi/romio/adio/ad_nfs/ad_nfs.c index b335cd4ae1e..73fdf214d83 100644 --- a/src/mpi/romio/adio/ad_nfs/ad_nfs.c +++ b/src/mpi/romio/adio/ad_nfs/ad_nfs.c @@ -32,6 +32,7 @@ struct ADIOI_Fns_struct ADIO_NFS_operations = { ADIOI_GEN_IreadStrided, /* IreadStrided */ ADIOI_GEN_IwriteStrided, /* IwriteStrided */ ADIOI_GEN_Flush, /* Flush */ + ADIOI_GEN_Fence, /* Fence */ ADIOI_NFS_Resize, /* Resize */ ADIOI_GEN_Delete, /* Delete */ ADIOI_NFS_Feature, /* Features */ diff --git a/src/mpi/romio/adio/ad_testfs/Makefile.mk b/src/mpi/romio/adio/ad_testfs/Makefile.mk index 342fa56c256..a33c38848ee 100644 --- a/src/mpi/romio/adio/ad_testfs/Makefile.mk +++ b/src/mpi/romio/adio/ad_testfs/Makefile.mk @@ -20,6 +20,7 @@ romio_other_sources += \ adio/ad_testfs/ad_testfs_iwrite.c \ adio/ad_testfs/ad_testfs_wait.c \ adio/ad_testfs/ad_testfs_flush.c \ + adio/ad_testfs/ad_testfs_fence.c \ adio/ad_testfs/ad_testfs_seek.c \ adio/ad_testfs/ad_testfs_resize.c \ adio/ad_testfs/ad_testfs_hints.c \ diff --git a/src/mpi/romio/adio/ad_testfs/ad_testfs.c b/src/mpi/romio/adio/ad_testfs/ad_testfs.c index 5e6a5a1df14..5d10a0a7310 100644 --- a/src/mpi/romio/adio/ad_testfs/ad_testfs.c +++ b/src/mpi/romio/adio/ad_testfs/ad_testfs.c @@ -30,6 +30,7 @@ struct ADIOI_Fns_struct ADIO_TESTFS_operations = { ADIOI_TESTFS_IreadStrided, /* IreadStrided */ ADIOI_TESTFS_IwriteStrided, /* IwriteStrided */ ADIOI_TESTFS_Flush, /* Flush */ + ADIOI_TESTFS_Fence, /* Fence */ ADIOI_TESTFS_Resize, /* Resize */ ADIOI_TESTFS_Delete, /* Delete */ ADIOI_GEN_Feature, /* Features */ diff --git a/src/mpi/romio/adio/ad_testfs/ad_testfs.h b/src/mpi/romio/adio/ad_testfs/ad_testfs.h index 0777da52959..9f5951b9005 100644 --- a/src/mpi/romio/adio/ad_testfs/ad_testfs.h +++ b/src/mpi/romio/adio/ad_testfs/ad_testfs.h @@ -62,6 +62,7 @@ void ADIOI_TESTFS_IwriteStrided(ADIO_File fd, const void *buf, int count, ADIO_Offset offset, ADIO_Request * request, int *error_code); void ADIOI_TESTFS_Flush(ADIO_File fd, int *error_code); +void ADIOI_TESTFS_Fence(ADIO_File fd, int *error_code); void ADIOI_TESTFS_Resize(ADIO_File fd, ADIO_Offset size, int *error_code); ADIO_Offset ADIOI_TESTFS_SeekIndividual(ADIO_File fd, ADIO_Offset offset, int whence, int *error_code); diff --git a/src/mpi/romio/adio/ad_testfs/ad_testfs_fence.c b/src/mpi/romio/adio/ad_testfs/ad_testfs_fence.c new file mode 100644 index 00000000000..5347c32a6b2 --- /dev/null +++ b/src/mpi/romio/adio/ad_testfs/ad_testfs_fence.c @@ -0,0 +1,18 @@ +/* + * Copyright (C) by Argonne National Laboratory + * See COPYRIGHT in top-level directory + */ + +#include "ad_testfs.h" +#include "adioi.h" + +void ADIOI_TESTFS_Fence(ADIO_File fd, int *error_code) +{ + int myrank, nprocs; + + *error_code = MPI_SUCCESS; + + MPI_Comm_size(fd->comm, &nprocs); + MPI_Comm_rank(fd->comm, &myrank); + FPRINTF(stdout, "[%d/%d] ADIOI_TESTFS_Fence called on %s\n", myrank, nprocs, fd->filename); +} diff --git a/src/mpi/romio/adio/ad_ufs/ad_ufs.c b/src/mpi/romio/adio/ad_ufs/ad_ufs.c index aac86d0b41f..f9107b0244f 100644 --- a/src/mpi/romio/adio/ad_ufs/ad_ufs.c +++ b/src/mpi/romio/adio/ad_ufs/ad_ufs.c @@ -35,6 +35,7 @@ struct ADIOI_Fns_struct ADIO_UFS_operations = { ADIOI_GEN_IreadStrided, /* IreadStrided */ ADIOI_GEN_IwriteStrided, /* IwriteStrided */ ADIOI_GEN_Flush, /* Flush */ + ADIOI_GEN_Fence, /* Fence */ ADIOI_GEN_Resize, /* Resize */ ADIOI_GEN_Delete, /* Delete */ ADIOI_GEN_Feature, /* Features */ diff --git a/src/mpi/romio/adio/ad_unify/Makefile.mk b/src/mpi/romio/adio/ad_unify/Makefile.mk index 6adf09824ee..71053ed6555 100644 --- a/src/mpi/romio/adio/ad_unify/Makefile.mk +++ b/src/mpi/romio/adio/ad_unify/Makefile.mk @@ -15,6 +15,7 @@ romio_other_sources += \ adio/ad_unify/ad_unify_close.c \ adio/ad_unify/ad_unify_io.c \ adio/ad_unify/ad_unify_flush.c \ + adio/ad_unify/ad_unify_fence.c \ adio/ad_unify/ad_unify_delete.c \ adio/ad_unify/ad_unify_fcntl.c \ adio/ad_unify/ad_unify_resize.c \ diff --git a/src/mpi/romio/adio/ad_unify/ad_unify.c b/src/mpi/romio/adio/ad_unify/ad_unify.c index 9ac2a8ddda2..c91569ccb25 100644 --- a/src/mpi/romio/adio/ad_unify/ad_unify.c +++ b/src/mpi/romio/adio/ad_unify/ad_unify.c @@ -35,6 +35,7 @@ struct ADIOI_Fns_struct ADIO_UNIFY_operations = { ADIOI_FAKE_IreadStrided, /* IreadStrided */ ADIOI_FAKE_IwriteStrided, /* IwriteStrided */ ADIOI_UNIFY_Flush, /* Flush */ + ADIOI_UNIFY_Fence, /* Fence */ ADIOI_UNIFY_Resize, /* Resize */ ADIOI_UNIFY_Delete, /* Delete */ ADIOI_UNIFY_Feature, diff --git a/src/mpi/romio/adio/ad_unify/ad_unify_fence.c b/src/mpi/romio/adio/ad_unify/ad_unify_fence.c new file mode 100644 index 00000000000..0e3c8d5a80e --- /dev/null +++ b/src/mpi/romio/adio/ad_unify/ad_unify_fence.c @@ -0,0 +1,35 @@ + +#include "ad_unify.h" +#include "ad_unify_common.h" + +void ADIOI_UNIFY_Fence(ADIO_File fd, int *error_code) +{ + int ret; + *error_code = MPI_SUCCESS; + ADIOI_UNIFY_fs *unifyfs_blob = fd->fs_ptr; + + /* Synchronize with file system to flush writes + * to make data visible to other processes. */ + unifyfs_io_request sync_req = { + .op = UNIFYFS_IOREQ_OP_SYNC_META, + .gfid = unifyfs_blob->gfid, + .result.error = 0, + }; + ret = unifyfs_dispatch_io(unifyfs_blob->fshdl, 1, &sync_req); + if (ret == UNIFYFS_SUCCESS) { + ret = unifyfs_wait_io(unifyfs_blob->fshdl, 1, &sync_req, 1); + if (ret == UNIFYFS_SUCCESS) + if (sync_req.result.error != 0) { + *error_code = MPIO_Err_create_code(MPI_SUCCESS, + MPIR_ERR_RECOVERABLE, __func__, __LINE__, + MPI_ERR_IO, "**io", "**io %s", + strerror(sync_req.result.error)); + } + } + + /* Synchronize processes so that upon returning from fence, + * all procs know that all other procs have completed their flush. */ + MPI_Barrier(fd->comm); + + return; +} diff --git a/src/mpi/romio/adio/common/Makefile.mk b/src/mpi/romio/adio/common/Makefile.mk index 47ee40dba04..c75a346cbd1 100644 --- a/src/mpi/romio/adio/common/Makefile.mk +++ b/src/mpi/romio/adio/common/Makefile.mk @@ -17,6 +17,7 @@ romio_other_sources += \ adio/common/ad_fcntl.c \ adio/common/ad_features.c \ adio/common/ad_flush.c \ + adio/common/ad_fence.c \ adio/common/ad_fstype.c \ adio/common/ad_get_sh_fp.c \ adio/common/ad_hints.c \ diff --git a/src/mpi/romio/adio/common/ad_fence.c b/src/mpi/romio/adio/common/ad_fence.c new file mode 100644 index 00000000000..ef84ed56a73 --- /dev/null +++ b/src/mpi/romio/adio/common/ad_fence.c @@ -0,0 +1,31 @@ +/* + * Copyright (C) by Argonne National Laboratory + * See COPYRIGHT in top-level directory + */ + +#include "adio.h" + +#ifdef HAVE_UNISTD_H +#include +#endif + +void ADIOI_GEN_Fence(ADIO_File fd, int *error_code) +{ + int err; + static char myname[] = "ADIOI_GEN_FENCE"; + + /* For the general (common) case, we assume that the underlying + * file system implements a POSIX-compliant write, + * meaning that written data becomes visible to other processes + * immediately upon returning from a successful write call. + * + * For such file systems, there is no need to explicitly flush + * data or to synchronize with the file system to expose the + * data from those write calls. */ + + /* Even though the data flush step is a NOP, + * fence semantics still require that we synchronize processes. */ + MPI_Barrier(fd->comm); + + *error_code = MPI_SUCCESS; +} diff --git a/src/mpi/romio/adio/include/adio.h b/src/mpi/romio/adio/include/adio.h index 3f6fe59d1db..c957d0ce34f 100644 --- a/src/mpi/romio/adio/include/adio.h +++ b/src/mpi/romio/adio/include/adio.h @@ -396,6 +396,7 @@ void ADIO_IwriteStridedColl(ADIO_File fd, void *buf, int count, ADIO_Offset ADIO_SeekIndividual(ADIO_File fd, ADIO_Offset offset, int whence, int *error_code); void ADIO_Delete(char *filename, int *error_code); void ADIO_Flush(ADIO_File fd, int *error_code); +void ADIO_Fence(ADIO_File fd, int *error_code); void ADIO_Resize(ADIO_File fd, ADIO_Offset size, int *error_code); void ADIO_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code); void ADIO_ResolveFileType(MPI_Comm comm, const char *filename, int *fstype, diff --git a/src/mpi/romio/adio/include/adioi.h b/src/mpi/romio/adio/include/adioi.h index 01300b459ca..309021181c7 100644 --- a/src/mpi/romio/adio/include/adioi.h +++ b/src/mpi/romio/adio/include/adioi.h @@ -226,6 +226,7 @@ struct ADIOI_Fns_struct { MPI_Datatype datatype, int file_ptr_type, ADIO_Offset offset, ADIO_Request * request, int *error_code); void (*ADIOI_xxx_Flush) (ADIO_File fd, int *error_code); + void (*ADIOI_xxx_Fence) (ADIO_File fd, int *error_code); void (*ADIOI_xxx_Resize) (ADIO_File fd, ADIO_Offset size, int *error_code); void (*ADIOI_xxx_Delete) (const char *filename, int *error_code); int (*ADIOI_xxx_Feature) (ADIO_File fd, int flag); @@ -332,6 +333,8 @@ struct ADIOI_Fns_struct { #define ADIO_Flush(fd,error_code) (*(fd->fns->ADIOI_xxx_Flush))(fd,error_code) +#define ADIO_Fence(fd,error_code) (*(fd->fns->ADIOI_xxx_Fence))(fd,error_code) + #define ADIO_Resize(fd,size,error_code) \ (*(fd->fns->ADIOI_xxx_Resize))(fd,size,error_code) @@ -395,6 +398,7 @@ void ADIOI_Info_print_keyvals(MPI_Info info); void ADIOI_GEN_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t * fcntl_struct, int *error_code); void ADIOI_GEN_Flush(ADIO_File fd, int *error_code); +void ADIOI_GEN_Fence(ADIO_File fd, int *error_code); void ADIOI_GEN_OpenColl(ADIO_File fd, int rank, int access_mode, int *error_code); void ADIOI_SCALEABLE_OpenColl(ADIO_File fd, int rank, int access_mode, int *error_code); void ADIOI_FAILSAFE_OpenColl(ADIO_File fd, int rank, int access_mode, int *error_code); diff --git a/src/mpi/romio/include/mpio.h.in b/src/mpi/romio/include/mpio.h.in index c0322fdb1ad..252bb5823ff 100644 --- a/src/mpi/romio/include/mpio.h.in +++ b/src/mpi/romio/include/mpio.h.in @@ -246,6 +246,7 @@ int MPI_Register_datarep(const char *datarep, MPI_Datarep_conversion_function *r int MPI_File_set_atomicity(MPI_File fh, int flag) ROMIO_API_PUBLIC; int MPI_File_get_atomicity(MPI_File fh, int *flag) ROMIO_API_PUBLIC; int MPI_File_sync(MPI_File fh) ROMIO_API_PUBLIC; +int MPI_File_fence(MPI_File fh) ROMIO_API_PUBLIC; /* Section 4.13.3 */ #ifndef MPICH @@ -549,6 +550,7 @@ int PMPI_Register_datarep(const char *, int PMPI_File_set_atomicity(MPI_File, int) ROMIO_API_PUBLIC; int PMPI_File_get_atomicity(MPI_File, int *) ROMIO_API_PUBLIC; int PMPI_File_sync(MPI_File) ROMIO_API_PUBLIC; +int PMPI_File_fence(MPI_File) ROMIO_API_PUBLIC; /* Section 4.13.3 */ #ifndef MPICH diff --git a/src/mpi/romio/mpi-io/Makefile.mk b/src/mpi/romio/mpi-io/Makefile.mk index 4005d8252c3..2d7420700a4 100644 --- a/src/mpi/romio/mpi-io/Makefile.mk +++ b/src/mpi/romio/mpi-io/Makefile.mk @@ -15,6 +15,7 @@ romio_mpi_sources += \ mpi-io/file_c2f.c \ mpi-io/file_f2c.c \ mpi-io/fsync.c \ + mpi-io/fence.c \ mpi-io/get_amode.c \ mpi-io/get_atom.c \ mpi-io/get_bytoff.c \ diff --git a/src/mpi/romio/mpi-io/fence.c b/src/mpi/romio/mpi-io/fence.c new file mode 100644 index 00000000000..2a876588b03 --- /dev/null +++ b/src/mpi/romio/mpi-io/fence.c @@ -0,0 +1,71 @@ +/* + * Copyright (C) by Argonne National Laboratory + * See COPYRIGHT in top-level directory + */ + +#include "mpioimpl.h" + +#ifdef HAVE_WEAK_SYMBOLS + +#if defined(HAVE_PRAGMA_WEAK) +#pragma weak MPI_File_fence = PMPI_File_fence +#elif defined(HAVE_PRAGMA_HP_SEC_DEF) +#pragma _HP_SECONDARY_DEF PMPI_File_fence MPI_File_fence +#elif defined(HAVE_PRAGMA_CRI_DUP) +#pragma _CRI duplicate MPI_File_fence as PMPI_File_fence +/* end of weak pragmas */ +#elif defined(HAVE_WEAK_ATTRIBUTE) +int MPI_File_fence(MPI_File fh) __attribute__ ((weak, alias("PMPI_File_fence"))); +#endif + +/* Include mapping from MPI->PMPI */ +#define MPIO_BUILD_PROFILING +#include "mpioprof.h" +#endif + +/*@ + MPI_File_fence - Causes all previous writes to become visible + to other processes + +Input Parameters: +. fh - file handle (handle) + +.N fortran +@*/ +int MPI_File_fence(MPI_File fh) +{ + int error_code; + ADIO_File adio_fh; + static char myname[] = "MPI_FILE_FENCE"; +#ifdef MPI_hpux + int fl_xmpi; + + HPMP_IO_START(fl_xmpi, BLKMPIFILEFENCE, TRDTBLOCK, adio_fh, MPI_DATATYPE_NULL, -1); +#endif /* MPI_hpux */ + ROMIO_THREAD_CS_ENTER(); + + adio_fh = MPIO_File_resolve(fh); + /* --BEGIN ERROR HANDLING-- */ + if ((adio_fh == NULL) || ((adio_fh)->cookie != ADIOI_FILE_COOKIE)) { + error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, + myname, __LINE__, MPI_ERR_ARG, "**iobadfh", 0); + error_code = MPIO_Err_return_file(MPI_FILE_NULL, error_code); + goto fn_exit; + } + MPIO_CHECK_WRITABLE(fh, myname, error_code); + /* --END ERROR HANDLING-- */ + + ADIO_Fence(adio_fh, &error_code); + /* --BEGIN ERROR HANDLING-- */ + if (error_code != MPI_SUCCESS) + error_code = MPIO_Err_return_file(adio_fh, error_code); + /* --END ERROR HANDLING-- */ + +#ifdef MPI_hpux + HPMP_IO_END(fl_xmpi, adio_fh, MPI_DATATYPE_NULL, -1); +#endif /* MPI_hpux */ + + fn_exit: + ROMIO_THREAD_CS_EXIT(); + return error_code; +} diff --git a/src/mpi/romio/mpi-io/mpioprof.h b/src/mpi/romio/mpi-io/mpioprof.h index c32f812479a..5e7455d06cd 100644 --- a/src/mpi/romio/mpi-io/mpioprof.h +++ b/src/mpi/romio/mpi-io/mpioprof.h @@ -139,6 +139,8 @@ #define MPI_File_get_atomicity PMPI_File_get_atomicity #undef MPI_File_sync #define MPI_File_sync PMPI_File_sync +#undef MPI_File_fence +#define MPI_File_fence PMPI_File_fence #undef MPI_Type_create_subarray #define MPI_Type_create_subarray PMPI_Type_create_subarray diff --git a/src/mpi/romio/test/perf.c b/src/mpi/romio/test/perf.c index e8c469d3e9e..7d025335900 100644 --- a/src/mpi/romio/test/perf.c +++ b/src/mpi/romio/test/perf.c @@ -158,6 +158,55 @@ int main(int argc, char **argv) } } + flag = 0; + for (j = 0; j < ntimes; j++) { + MPI_CHECK(MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | + MPI_MODE_RDWR, MPI_INFO_NULL, &fh)); + MPI_CHECK(MPI_File_seek(fh, mynod * SIZE, MPI_SEEK_SET)); + + MPI_Barrier(MPI_COMM_WORLD); + stim = MPI_Wtime(); + MPI_CHECK(MPI_File_write(fh, buf, SIZE, MPI_BYTE, &status)); + err = MPI_File_fence(fh); + write_tim = MPI_Wtime() - stim; + if (err == MPI_ERR_UNKNOWN) { + flag = 1; + break; + } + + MPI_CHECK(MPI_File_close(&fh)); + + MPI_Barrier(MPI_COMM_WORLD); + + MPI_CHECK(MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_CREATE | + MPI_MODE_RDWR, MPI_INFO_NULL, &fh)); + MPI_CHECK(MPI_File_seek(fh, mynod * SIZE, MPI_SEEK_SET)); + + MPI_Barrier(MPI_COMM_WORLD); + stim = MPI_Wtime(); + MPI_CHECK(MPI_File_read(fh, buf, SIZE, MPI_BYTE, &status)); + read_tim = MPI_Wtime() - stim; + + MPI_CHECK(MPI_File_close(&fh)); + + MPI_Allreduce(&write_tim, &new_write_tim, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + MPI_Allreduce(&read_tim, &new_read_tim, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + + min_read_tim = (new_read_tim < min_read_tim) ? new_read_tim : min_read_tim; + min_write_tim = (new_write_tim < min_write_tim) ? new_write_tim : min_write_tim; + } + + if (mynod == 0) { + if (flag) + fprintf(stderr, "MPI_File_sync returns error.\n"); + else { + read_bw = (SIZE * nprocs) / (min_read_tim * 1024.0 * 1024.0); + write_bw = (SIZE * nprocs) / (min_write_tim * 1024.0 * 1024.0); + fprintf(stderr, "Write bandwidth including file fence = %f Mbytes/sec\n", write_bw); + fprintf(stderr, "Read bandwidth after file fence = %f Mbytes/sec\n", read_bw); + } + } + free(buf); free(filename); MPI_Finalize();